2024-11-19 04:40:27,211 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 04:40:27,227 main DEBUG Took 0.013383 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 04:40:27,228 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 04:40:27,228 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 04:40:27,230 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 04:40:27,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,251 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 04:40:27,268 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,271 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,272 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,272 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,273 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,274 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,274 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,274 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,276 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,277 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,278 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,279 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,280 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,281 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 04:40:27,283 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,283 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 04:40:27,285 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 04:40:27,287 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 04:40:27,289 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 04:40:27,290 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 04:40:27,291 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 04:40:27,292 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 04:40:27,303 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 04:40:27,306 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 04:40:27,308 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 04:40:27,308 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 04:40:27,309 main DEBUG createAppenders(={Console}) 2024-11-19 04:40:27,310 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 04:40:27,310 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 04:40:27,311 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 04:40:27,311 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 04:40:27,312 main DEBUG OutputStream closed 2024-11-19 04:40:27,312 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 04:40:27,313 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 04:40:27,313 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 04:40:27,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 04:40:27,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 04:40:27,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 04:40:27,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 04:40:27,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 04:40:27,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 04:40:27,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 04:40:27,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 04:40:27,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 04:40:27,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 04:40:27,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 04:40:27,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 04:40:27,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 04:40:27,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 04:40:27,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 04:40:27,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 04:40:27,458 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 04:40:27,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 04:40:27,462 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 04:40:27,462 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 04:40:27,463 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 04:40:27,464 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T04:40:27,850 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c 2024-11-19 04:40:27,855 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 04:40:27,856 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T04:40:27,869 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-19T04:40:27,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T04:40:27,942 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3, deleteOnExit=true 2024-11-19T04:40:27,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T04:40:27,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/test.cache.data in system properties and HBase conf 2024-11-19T04:40:27,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T04:40:27,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/hadoop.log.dir in system properties and HBase conf 2024-11-19T04:40:27,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T04:40:27,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T04:40:27,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T04:40:28,076 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T04:40:28,215 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T04:40:28,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T04:40:28,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T04:40:28,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T04:40:28,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T04:40:28,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T04:40:28,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T04:40:28,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T04:40:28,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T04:40:28,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T04:40:28,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/nfs.dump.dir in system properties and HBase conf 2024-11-19T04:40:28,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/java.io.tmpdir in system properties and HBase conf 2024-11-19T04:40:28,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T04:40:28,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T04:40:28,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T04:40:29,177 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T04:40:29,267 INFO [Time-limited test {}] log.Log(170): Logging initialized @3046ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T04:40:29,365 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T04:40:29,442 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T04:40:29,477 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T04:40:29,478 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T04:40:29,479 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T04:40:29,494 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T04:40:29,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/hadoop.log.dir/,AVAILABLE} 2024-11-19T04:40:29,497 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T04:40:29,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/java.io.tmpdir/jetty-localhost-40295-hadoop-hdfs-3_4_1-tests_jar-_-any-11101451876379313306/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T04:40:29,751 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40295} 2024-11-19T04:40:29,751 INFO [Time-limited test {}] server.Server(415): Started @3532ms 2024-11-19T04:40:30,185 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T04:40:30,195 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T04:40:30,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T04:40:30,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T04:40:30,197 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T04:40:30,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/hadoop.log.dir/,AVAILABLE} 2024-11-19T04:40:30,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T04:40:30,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/java.io.tmpdir/jetty-localhost-39097-hadoop-hdfs-3_4_1-tests_jar-_-any-14107641279146365432/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T04:40:30,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39097} 2024-11-19T04:40:30,330 INFO [Time-limited test {}] server.Server(415): Started @4111ms 2024-11-19T04:40:30,393 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T04:40:31,004 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/data/data1/current/BP-1974752605-172.17.0.2-1731991228892/current, will proceed with Du for space computation calculation, 2024-11-19T04:40:31,011 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/data/data2/current/BP-1974752605-172.17.0.2-1731991228892/current, will proceed with Du for space computation calculation, 2024-11-19T04:40:31,090 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T04:40:31,177 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80f969ec3ff6a3b with lease ID 0x705c69917c53d7f8: Processing first storage report for DS-cd5b4666-cb7f-4ef9-979a-bf618b01e6c0 from datanode DatanodeRegistration(127.0.0.1:38523, datanodeUuid=5cca3b6b-f189-46f5-842b-0fb0bedb6747, infoPort=44375, infoSecurePort=0, ipcPort=44723, storageInfo=lv=-57;cid=testClusterID;nsid=1519192921;c=1731991228892) 2024-11-19T04:40:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80f969ec3ff6a3b with lease ID 0x705c69917c53d7f8: from storage DS-cd5b4666-cb7f-4ef9-979a-bf618b01e6c0 node DatanodeRegistration(127.0.0.1:38523, datanodeUuid=5cca3b6b-f189-46f5-842b-0fb0bedb6747, infoPort=44375, infoSecurePort=0, ipcPort=44723, storageInfo=lv=-57;cid=testClusterID;nsid=1519192921;c=1731991228892), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T04:40:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80f969ec3ff6a3b with lease ID 0x705c69917c53d7f8: Processing first storage report for DS-890fbd37-6e8d-4d5d-a76d-50b5850406ff from datanode DatanodeRegistration(127.0.0.1:38523, datanodeUuid=5cca3b6b-f189-46f5-842b-0fb0bedb6747, infoPort=44375, infoSecurePort=0, ipcPort=44723, storageInfo=lv=-57;cid=testClusterID;nsid=1519192921;c=1731991228892) 2024-11-19T04:40:31,180 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80f969ec3ff6a3b with lease ID 0x705c69917c53d7f8: from storage DS-890fbd37-6e8d-4d5d-a76d-50b5850406ff node DatanodeRegistration(127.0.0.1:38523, datanodeUuid=5cca3b6b-f189-46f5-842b-0fb0bedb6747, infoPort=44375, infoSecurePort=0, ipcPort=44723, storageInfo=lv=-57;cid=testClusterID;nsid=1519192921;c=1731991228892), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T04:40:31,209 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c 2024-11-19T04:40:31,330 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/zookeeper_0, clientPort=56501, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T04:40:31,346 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56501 2024-11-19T04:40:31,364 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:31,368 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:31,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741825_1001 (size=7) 2024-11-19T04:40:32,048 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 with version=8 2024-11-19T04:40:32,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/hbase-staging 2024-11-19T04:40:32,145 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T04:40:32,412 INFO [Time-limited test {}] client.ConnectionUtils(128): master/08a7f35e60d4:0 server-side Connection retries=45 2024-11-19T04:40:32,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T04:40:32,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T04:40:32,428 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T04:40:32,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T04:40:32,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T04:40:32,593 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T04:40:32,661 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T04:40:32,671 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T04:40:32,675 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T04:40:32,703 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1554 (auto-detected) 2024-11-19T04:40:32,704 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T04:40:32,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46599 2024-11-19T04:40:32,763 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46599 connecting to ZooKeeper ensemble=127.0.0.1:56501 2024-11-19T04:40:32,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:465990x0, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T04:40:32,805 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46599-0x1012e86c5d20000 connected 2024-11-19T04:40:32,857 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:32,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:32,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T04:40:32,882 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04, hbase.cluster.distributed=false 2024-11-19T04:40:32,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T04:40:32,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46599 2024-11-19T04:40:32,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46599 2024-11-19T04:40:32,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46599 2024-11-19T04:40:32,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46599 2024-11-19T04:40:32,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46599 2024-11-19T04:40:33,066 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/08a7f35e60d4:0 server-side Connection retries=45 2024-11-19T04:40:33,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T04:40:33,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T04:40:33,068 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T04:40:33,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T04:40:33,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T04:40:33,072 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T04:40:33,075 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T04:40:33,076 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32975 2024-11-19T04:40:33,078 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32975 connecting to ZooKeeper ensemble=127.0.0.1:56501 2024-11-19T04:40:33,080 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:33,084 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:33,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329750x0, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T04:40:33,091 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32975-0x1012e86c5d20001 connected 2024-11-19T04:40:33,091 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T04:40:33,097 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T04:40:33,106 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T04:40:33,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T04:40:33,115 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T04:40:33,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32975 2024-11-19T04:40:33,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32975 2024-11-19T04:40:33,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32975 2024-11-19T04:40:33,119 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32975 2024-11-19T04:40:33,120 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32975 2024-11-19T04:40:33,139 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;08a7f35e60d4:46599 2024-11-19T04:40:33,140 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:33,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T04:40:33,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T04:40:33,152 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:33,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:33,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T04:40:33,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:33,181 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T04:40:33,182 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/08a7f35e60d4,46599,1731991232199 from backup master directory 2024-11-19T04:40:33,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:33,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T04:40:33,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T04:40:33,187 WARN [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T04:40:33,187 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:33,189 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T04:40:33,191 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T04:40:33,263 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/hbase.id] with ID: f524422f-86ed-4a7c-aeb0-e6f8c33d39a3 2024-11-19T04:40:33,263 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/.tmp/hbase.id 2024-11-19T04:40:33,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741826_1002 (size=42) 2024-11-19T04:40:33,677 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/.tmp/hbase.id]:[hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/hbase.id] 2024-11-19T04:40:33,735 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T04:40:33,742 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T04:40:33,764 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-19T04:40:33,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:33,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:33,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741827_1003 (size=196) 2024-11-19T04:40:33,802 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:40:33,805 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T04:40:33,825 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:33,830 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T04:40:33,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741828_1004 (size=1189) 2024-11-19T04:40:34,303 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store 2024-11-19T04:40:34,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741829_1005 (size=34) 2024-11-19T04:40:34,349 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T04:40:34,354 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:40:34,356 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T04:40:34,356 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:40:34,357 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:40:34,359 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T04:40:34,359 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:40:34,360 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:40:34,362 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731991234356Disabling compacts and flushes for region at 1731991234356Disabling writes for close at 1731991234359 (+3 ms)Writing region close event to WAL at 1731991234359Closed at 1731991234359 2024-11-19T04:40:34,365 WARN [master/08a7f35e60d4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/.initializing 2024-11-19T04:40:34,365 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/WALs/08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:34,376 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T04:40:34,395 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=08a7f35e60d4%2C46599%2C1731991232199, suffix=, logDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/WALs/08a7f35e60d4,46599,1731991232199, archiveDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/oldWALs, maxLogs=10 2024-11-19T04:40:34,424 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/WALs/08a7f35e60d4,46599,1731991232199/08a7f35e60d4%2C46599%2C1731991232199.1731991234400, exclude list is [], retry=0 2024-11-19T04:40:34,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38523,DS-cd5b4666-cb7f-4ef9-979a-bf618b01e6c0,DISK] 2024-11-19T04:40:34,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T04:40:34,489 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/WALs/08a7f35e60d4,46599,1731991232199/08a7f35e60d4%2C46599%2C1731991232199.1731991234400 2024-11-19T04:40:34,490 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44375:44375)] 2024-11-19T04:40:34,491 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:40:34,491 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:40:34,495 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,496 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T04:40:34,569 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:34,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:34,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T04:40:34,576 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:34,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:40:34,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T04:40:34,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:34,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:40:34,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T04:40:34,585 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:34,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:40:34,586 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,589 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,590 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,595 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,596 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,599 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T04:40:34,602 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T04:40:34,607 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:40:34,609 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65237158, jitterRate=-0.02789059281349182}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T04:40:34,615 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731991234511Initializing all the Stores at 1731991234514 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991234514Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991234515 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991234515Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991234516 (+1 ms)Cleaning up temporary data from old regions at 1731991234596 (+80 ms)Region opened successfully at 1731991234615 (+19 ms) 2024-11-19T04:40:34,617 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T04:40:34,660 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77884a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=08a7f35e60d4/172.17.0.2:0 2024-11-19T04:40:34,699 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T04:40:34,711 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T04:40:34,711 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T04:40:34,716 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T04:40:34,717 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T04:40:34,723 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T04:40:34,724 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T04:40:34,757 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T04:40:34,767 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T04:40:34,770 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T04:40:34,773 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T04:40:34,774 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T04:40:34,776 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T04:40:34,778 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T04:40:34,782 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T04:40:34,785 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T04:40:34,787 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T04:40:34,789 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T04:40:34,810 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T04:40:34,811 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T04:40:34,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T04:40:34,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T04:40:34,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:34,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:34,819 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=08a7f35e60d4,46599,1731991232199, sessionid=0x1012e86c5d20000, setting cluster-up flag (Was=false) 2024-11-19T04:40:34,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:34,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:34,844 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T04:40:34,846 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:34,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:34,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:34,859 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T04:40:34,861 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:34,868 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T04:40:34,927 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(746): ClusterId : f524422f-86ed-4a7c-aeb0-e6f8c33d39a3 2024-11-19T04:40:34,930 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T04:40:34,936 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T04:40:34,936 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T04:40:34,941 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T04:40:34,942 DEBUG [RS:0;08a7f35e60d4:32975 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56186748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=08a7f35e60d4/172.17.0.2:0 2024-11-19T04:40:34,956 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T04:40:34,962 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;08a7f35e60d4:32975 2024-11-19T04:40:34,965 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T04:40:34,965 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T04:40:34,965 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T04:40:34,968 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(2659): reportForDuty to master=08a7f35e60d4,46599,1731991232199 with port=32975, startcode=1731991233019 2024-11-19T04:40:34,969 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T04:40:34,975 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T04:40:34,981 DEBUG [RS:0;08a7f35e60d4:32975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T04:40:34,981 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 08a7f35e60d4,46599,1731991232199 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T04:40:34,988 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/08a7f35e60d4:0, corePoolSize=5, maxPoolSize=5 2024-11-19T04:40:34,988 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/08a7f35e60d4:0, corePoolSize=5, maxPoolSize=5 2024-11-19T04:40:34,988 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/08a7f35e60d4:0, corePoolSize=5, maxPoolSize=5 2024-11-19T04:40:34,988 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/08a7f35e60d4:0, corePoolSize=5, maxPoolSize=5 2024-11-19T04:40:34,988 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/08a7f35e60d4:0, corePoolSize=10, maxPoolSize=10 2024-11-19T04:40:34,989 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:34,989 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/08a7f35e60d4:0, corePoolSize=2, maxPoolSize=2 2024-11-19T04:40:34,989 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:34,993 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731991264993 2024-11-19T04:40:34,994 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T04:40:34,995 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T04:40:34,995 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T04:40:34,996 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T04:40:35,000 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T04:40:35,001 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T04:40:35,001 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T04:40:35,001 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T04:40:35,002 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:35,002 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T04:40:35,008 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,012 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T04:40:35,014 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T04:40:35,014 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T04:40:35,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741831_1007 (size=1321) 2024-11-19T04:40:35,017 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T04:40:35,017 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T04:40:35,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40331, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T04:40:35,020 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/08a7f35e60d4:0:becomeActiveMaster-HFileCleaner.large.0-1731991235020,5,FailOnTimeoutGroup] 2024-11-19T04:40:35,021 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/08a7f35e60d4:0:becomeActiveMaster-HFileCleaner.small.0-1731991235020,5,FailOnTimeoutGroup] 2024-11-19T04:40:35,021 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,021 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T04:40:35,023 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,023 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,026 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:35,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46599 {}] master.ServerManager(517): Registering regionserver=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:35,043 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:40:35,043 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34817 2024-11-19T04:40:35,043 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T04:40:35,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T04:40:35,048 DEBUG [RS:0;08a7f35e60d4:32975 {}] zookeeper.ZKUtil(111): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:35,049 WARN [RS:0;08a7f35e60d4:32975 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T04:40:35,049 INFO [RS:0;08a7f35e60d4:32975 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T04:40:35,049 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:35,051 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [08a7f35e60d4,32975,1731991233019] 2024-11-19T04:40:35,075 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T04:40:35,088 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T04:40:35,093 INFO [RS:0;08a7f35e60d4:32975 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T04:40:35,093 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,094 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T04:40:35,100 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T04:40:35,101 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,101 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,102 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,102 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,102 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,102 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,102 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/08a7f35e60d4:0, corePoolSize=2, maxPoolSize=2 2024-11-19T04:40:35,102 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,103 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,103 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,103 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,103 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,103 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/08a7f35e60d4:0, corePoolSize=1, maxPoolSize=1 2024-11-19T04:40:35,103 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/08a7f35e60d4:0, corePoolSize=3, maxPoolSize=3 2024-11-19T04:40:35,104 DEBUG [RS:0;08a7f35e60d4:32975 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0, corePoolSize=3, maxPoolSize=3 2024-11-19T04:40:35,105 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,105 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,105 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,105 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,105 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,105 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,32975,1731991233019-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T04:40:35,125 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T04:40:35,128 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,32975,1731991233019-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,128 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,128 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.Replication(171): 08a7f35e60d4,32975,1731991233019 started 2024-11-19T04:40:35,152 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:35,153 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1482): Serving as 08a7f35e60d4,32975,1731991233019, RpcServer on 08a7f35e60d4/172.17.0.2:32975, sessionid=0x1012e86c5d20001 2024-11-19T04:40:35,154 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T04:40:35,154 DEBUG [RS:0;08a7f35e60d4:32975 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:35,154 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '08a7f35e60d4,32975,1731991233019' 2024-11-19T04:40:35,154 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T04:40:35,155 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T04:40:35,156 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T04:40:35,156 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T04:40:35,156 DEBUG [RS:0;08a7f35e60d4:32975 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:35,156 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '08a7f35e60d4,32975,1731991233019' 2024-11-19T04:40:35,156 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T04:40:35,157 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T04:40:35,158 DEBUG [RS:0;08a7f35e60d4:32975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T04:40:35,158 INFO [RS:0;08a7f35e60d4:32975 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T04:40:35,158 INFO [RS:0;08a7f35e60d4:32975 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T04:40:35,263 INFO [RS:0;08a7f35e60d4:32975 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T04:40:35,267 INFO [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=08a7f35e60d4%2C32975%2C1731991233019, suffix=, logDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019, archiveDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/oldWALs, maxLogs=32 2024-11-19T04:40:35,283 DEBUG [RS:0;08a7f35e60d4:32975 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019/08a7f35e60d4%2C32975%2C1731991233019.1731991235269, exclude list is [], retry=0 2024-11-19T04:40:35,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38523,DS-cd5b4666-cb7f-4ef9-979a-bf618b01e6c0,DISK] 2024-11-19T04:40:35,291 INFO [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019/08a7f35e60d4%2C32975%2C1731991233019.1731991235269 2024-11-19T04:40:35,291 DEBUG [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44375:44375)] 2024-11-19T04:40:35,418 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T04:40:35,418 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:40:35,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741833_1009 (size=32) 2024-11-19T04:40:35,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:40:35,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T04:40:35,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T04:40:35,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:35,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:35,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T04:40:35,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T04:40:35,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:35,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:35,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T04:40:35,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T04:40:35,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:35,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:35,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T04:40:35,846 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T04:40:35,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:35,847 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:35,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T04:40:35,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740 2024-11-19T04:40:35,850 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740 2024-11-19T04:40:35,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T04:40:35,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T04:40:35,853 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:40:35,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T04:40:35,859 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:40:35,860 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66290044, jitterRate=-0.012201368808746338}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:40:35,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731991235830Initializing all the Stores at 1731991235832 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991235832Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991235833 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991235833Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991235833Cleaning up temporary data from old regions at 1731991235853 (+20 ms)Region opened successfully at 1731991235862 (+9 ms) 2024-11-19T04:40:35,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T04:40:35,863 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T04:40:35,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T04:40:35,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T04:40:35,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T04:40:35,864 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T04:40:35,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731991235863Disabling compacts and flushes for region at 1731991235863Disabling writes for close at 1731991235863Writing region close event to WAL at 1731991235864 (+1 ms)Closed at 1731991235864 2024-11-19T04:40:35,868 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T04:40:35,868 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T04:40:35,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T04:40:35,882 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T04:40:35,885 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T04:40:36,038 DEBUG [08a7f35e60d4:46599 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T04:40:36,050 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:36,058 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 08a7f35e60d4,32975,1731991233019, state=OPENING 2024-11-19T04:40:36,063 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T04:40:36,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:36,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:40:36,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T04:40:36,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T04:40:36,067 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T04:40:36,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:40:36,245 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T04:40:36,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60115, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T04:40:36,260 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T04:40:36,260 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T04:40:36,261 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T04:40:36,264 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=08a7f35e60d4%2C32975%2C1731991233019.meta, suffix=.meta, logDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019, archiveDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/oldWALs, maxLogs=32 2024-11-19T04:40:36,281 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019/08a7f35e60d4%2C32975%2C1731991233019.meta.1731991236266.meta, exclude list is [], retry=0 2024-11-19T04:40:36,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38523,DS-cd5b4666-cb7f-4ef9-979a-bf618b01e6c0,DISK] 2024-11-19T04:40:36,288 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/WALs/08a7f35e60d4,32975,1731991233019/08a7f35e60d4%2C32975%2C1731991233019.meta.1731991236266.meta 2024-11-19T04:40:36,288 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44375:44375)] 2024-11-19T04:40:36,289 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:40:36,290 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T04:40:36,293 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T04:40:36,298 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T04:40:36,302 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T04:40:36,303 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:40:36,303 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T04:40:36,303 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T04:40:36,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T04:40:36,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T04:40:36,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:36,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:36,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T04:40:36,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T04:40:36,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:36,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:36,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T04:40:36,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T04:40:36,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:36,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:36,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T04:40:36,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T04:40:36,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:36,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T04:40:36,316 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T04:40:36,318 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740 2024-11-19T04:40:36,321 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740 2024-11-19T04:40:36,323 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T04:40:36,323 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T04:40:36,324 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:40:36,327 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T04:40:36,328 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74494312, jitterRate=0.11005175113677979}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:40:36,328 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T04:40:36,330 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731991236304Writing region info on filesystem at 1731991236304Initializing all the Stores at 1731991236306 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991236306Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991236306Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991236306Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731991236306Cleaning up temporary data from old regions at 1731991236323 (+17 ms)Running coprocessor post-open hooks at 1731991236328 (+5 ms)Region opened successfully at 1731991236330 (+2 ms) 2024-11-19T04:40:36,337 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731991236236 2024-11-19T04:40:36,348 DEBUG [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T04:40:36,349 INFO [RS_OPEN_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T04:40:36,350 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:36,352 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 08a7f35e60d4,32975,1731991233019, state=OPEN 2024-11-19T04:40:36,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T04:40:36,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T04:40:36,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T04:40:36,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T04:40:36,360 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:36,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T04:40:36,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=08a7f35e60d4,32975,1731991233019 in 292 msec 2024-11-19T04:40:36,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T04:40:36,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 492 msec 2024-11-19T04:40:36,372 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T04:40:36,372 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T04:40:36,392 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:36,394 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:36,414 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:36,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54669, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:36,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5360 sec 2024-11-19T04:40:36,437 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731991236437, completionTime=-1 2024-11-19T04:40:36,441 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T04:40:36,441 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T04:40:36,494 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-19T04:40:36,494 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731991296494 2024-11-19T04:40:36,494 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731991356494 2024-11-19T04:40:36,494 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 52 msec 2024-11-19T04:40:36,496 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,46599,1731991232199-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,497 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,46599,1731991232199-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,497 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,46599,1731991232199-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,499 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-08a7f35e60d4:46599, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,499 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,499 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,505 DEBUG [master/08a7f35e60d4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T04:40:36,526 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.339sec 2024-11-19T04:40:36,528 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T04:40:36,529 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T04:40:36,530 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T04:40:36,530 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T04:40:36,530 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T04:40:36,531 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,46599,1731991232199-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T04:40:36,532 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,46599,1731991232199-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T04:40:36,540 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T04:40:36,541 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T04:40:36,541 INFO [master/08a7f35e60d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=08a7f35e60d4,46599,1731991232199-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T04:40:36,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:36,639 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T04:40:36,639 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T04:40:36,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:36,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:36,655 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ecb092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:36,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:36,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:36,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:36,673 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:36,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:36,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:36,726 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=12884 2024-11-19T04:40:36,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:40:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13ee3c32 2024-11-19T04:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:40:36,746 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:40:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:40:36,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:40:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T04:40:36,764 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T04:40:36,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T04:40:36,766 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:36,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T04:40:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T04:40:36,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741835_1011 (size=1061) 2024-11-19T04:40:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T04:40:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T04:40:37,183 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:40:37,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741836_1012 (size=53) 2024-11-19T04:40:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T04:40:37,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:40:37,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e97938390deb1164b875a27699276aad, disabling compactions & flushes 2024-11-19T04:40:37,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. after waiting 0 ms 2024-11-19T04:40:37,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e97938390deb1164b875a27699276aad: Waiting for close lock at 1731991237593Disabling compacts and flushes for region at 1731991237593Disabling writes for close at 1731991237593Writing region close event to WAL at 1731991237593Closed at 1731991237593 2024-11-19T04:40:37,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T04:40:37,601 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731991237596"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731991237596"}]},"ts":"1731991237596"} 2024-11-19T04:40:37,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T04:40:37,607 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T04:40:37,610 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991237607"}]},"ts":"1731991237607"} 2024-11-19T04:40:37,614 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T04:40:37,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, ASSIGN}] 2024-11-19T04:40:37,620 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, ASSIGN 2024-11-19T04:40:37,621 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, ASSIGN; state=OFFLINE, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=false 2024-11-19T04:40:37,773 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e97938390deb1164b875a27699276aad, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:37,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, ASSIGN because future has completed 2024-11-19T04:40:37,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T04:40:37,940 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,940 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:40:37,941 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,941 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:40:37,941 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,941 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,943 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,947 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:40:37,948 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e97938390deb1164b875a27699276aad columnFamilyName A 2024-11-19T04:40:37,948 DEBUG [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:37,950 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.HStore(327): Store=e97938390deb1164b875a27699276aad/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:40:37,950 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,952 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:40:37,952 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e97938390deb1164b875a27699276aad columnFamilyName B 2024-11-19T04:40:37,953 DEBUG [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:37,954 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.HStore(327): Store=e97938390deb1164b875a27699276aad/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:40:37,954 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,956 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:40:37,956 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e97938390deb1164b875a27699276aad columnFamilyName C 2024-11-19T04:40:37,957 DEBUG [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:40:37,957 INFO [StoreOpener-e97938390deb1164b875a27699276aad-1 {}] regionserver.HStore(327): Store=e97938390deb1164b875a27699276aad/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:40:37,958 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,958 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,959 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,960 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,962 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,962 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,963 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:40:37,965 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,969 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:40:37,970 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e97938390deb1164b875a27699276aad; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60650452, jitterRate=-0.09623783826828003}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:40:37,970 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e97938390deb1164b875a27699276aad 2024-11-19T04:40:37,971 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e97938390deb1164b875a27699276aad: Running coprocessor pre-open hook at 1731991237941Writing region info on filesystem at 1731991237941Initializing all the Stores at 1731991237943 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991237943Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991237943Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991237943Cleaning up temporary data from old regions at 1731991237962 (+19 ms)Running coprocessor post-open hooks at 1731991237970 (+8 ms)Region opened successfully at 1731991237971 (+1 ms) 2024-11-19T04:40:37,973 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., pid=6, masterSystemTime=1731991237933 2024-11-19T04:40:37,977 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,977 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:37,978 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e97938390deb1164b875a27699276aad, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:37,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:40:37,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T04:40:37,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 in 206 msec 2024-11-19T04:40:37,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T04:40:37,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, ASSIGN in 371 msec 2024-11-19T04:40:37,992 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T04:40:37,993 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991237993"}]},"ts":"1731991237993"} 2024-11-19T04:40:37,996 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T04:40:37,997 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T04:40:38,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2420 sec 2024-11-19T04:40:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T04:40:38,919 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:38,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24219159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3feb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,932 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:38,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:38,938 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:40:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:40:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f451fdf 2024-11-19T04:40:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:40:38,944 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:40:38,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,951 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,954 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,957 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,962 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,973 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,976 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e0b13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,978 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@831686b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,980 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43df1407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15646188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638af609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7788dec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ec50e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,992 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc64f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,994 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:40:38,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:40:38,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:40:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:40:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:40:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:40:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:40:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:38,999 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:40:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280a2bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:40:39,006 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,006 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,006 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,018 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,021 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,024 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,037 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,039 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,043 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:40:39,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,047 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:40:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,048 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:40:39,050 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:39,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:40:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T04:40:39,058 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:40:39,061 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T04:40:39,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:39,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:39,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:39,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:39,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:39,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T04:40:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991299211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T04:40:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:39,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991299221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991299226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991299227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991299221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/2fea7607fa0c4c88bec9112863f18d11 is 50, key is test_row_0/A:col10/1731991239109/Put/seqid=0 2024-11-19T04:40:39,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741837_1013 (size=12001) 2024-11-19T04:40:39,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/2fea7607fa0c4c88bec9112863f18d11 2024-11-19T04:40:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T04:40:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991299379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991299379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991299381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991299382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991299382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e2e740689cef47d1b036711ccfd76bba is 50, key is test_row_0/B:col10/1731991239109/Put/seqid=0 2024-11-19T04:40:39,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T04:40:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741838_1014 (size=12001) 2024-11-19T04:40:39,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e2e740689cef47d1b036711ccfd76bba 2024-11-19T04:40:39,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/b477cc5a8578479ca3ddf7999957fd6a is 50, key is test_row_0/C:col10/1731991239109/Put/seqid=0 2024-11-19T04:40:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741839_1015 (size=12001) 2024-11-19T04:40:39,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/b477cc5a8578479ca3ddf7999957fd6a 2024-11-19T04:40:39,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/2fea7607fa0c4c88bec9112863f18d11 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2fea7607fa0c4c88bec9112863f18d11 2024-11-19T04:40:39,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2fea7607fa0c4c88bec9112863f18d11, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T04:40:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T04:40:39,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e2e740689cef47d1b036711ccfd76bba as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e2e740689cef47d1b036711ccfd76bba 2024-11-19T04:40:39,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T04:40:39,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e2e740689cef47d1b036711ccfd76bba, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T04:40:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/b477cc5a8578479ca3ddf7999957fd6a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b477cc5a8578479ca3ddf7999957fd6a 2024-11-19T04:40:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b477cc5a8578479ca3ddf7999957fd6a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-19T04:40:39,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e97938390deb1164b875a27699276aad in 507ms, sequenceid=14, compaction requested=false 2024-11-19T04:40:39,622 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T04:40:39,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:39,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T04:40:39,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:39,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c3da1fc79b8045b691c76b36eda77dfb is 50, key is test_row_0/A:col10/1731991239625/Put/seqid=0 2024-11-19T04:40:39,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991299666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741840_1016 (size=14341) 2024-11-19T04:40:39,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c3da1fc79b8045b691c76b36eda77dfb 2024-11-19T04:40:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T04:40:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991299669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991299671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991299674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991299677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b64b5d3bc52a41b9888704e3f9d01927 is 50, key is test_row_0/B:col10/1731991239625/Put/seqid=0 2024-11-19T04:40:39,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741841_1017 (size=12001) 2024-11-19T04:40:39,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b64b5d3bc52a41b9888704e3f9d01927 2024-11-19T04:40:39,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T04:40:39,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:39,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:39,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/e07d76691512480384ceee54dc78cf2b is 50, key is test_row_0/C:col10/1731991239625/Put/seqid=0 2024-11-19T04:40:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991299788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741842_1018 (size=12001) 2024-11-19T04:40:39,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/e07d76691512480384ceee54dc78cf2b 2024-11-19T04:40:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c3da1fc79b8045b691c76b36eda77dfb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3da1fc79b8045b691c76b36eda77dfb 2024-11-19T04:40:39,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991299797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991299809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991299809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991299809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:39,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3da1fc79b8045b691c76b36eda77dfb, entries=200, sequenceid=35, filesize=14.0 K 2024-11-19T04:40:39,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b64b5d3bc52a41b9888704e3f9d01927 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b64b5d3bc52a41b9888704e3f9d01927 2024-11-19T04:40:39,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b64b5d3bc52a41b9888704e3f9d01927, entries=150, sequenceid=35, filesize=11.7 K 2024-11-19T04:40:39,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/e07d76691512480384ceee54dc78cf2b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e07d76691512480384ceee54dc78cf2b 2024-11-19T04:40:39,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e07d76691512480384ceee54dc78cf2b, entries=150, sequenceid=35, filesize=11.7 K 2024-11-19T04:40:39,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e97938390deb1164b875a27699276aad in 227ms, sequenceid=35, compaction requested=false 2024-11-19T04:40:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:39,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T04:40:39,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:39,919 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:40:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:39,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:39,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/309af6ddd91c4e03981e858628e36dd1 is 50, key is test_row_0/A:col10/1731991239666/Put/seqid=0 2024-11-19T04:40:39,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741843_1019 (size=12001) 2024-11-19T04:40:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:40,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:40,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991240064, max timestamp=1731991240086Num uniques 859; 2024-11-19T04:40:40,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:40:40,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991240064, max timestamp=1731991240086Num uniques 859; 2024-11-19T04:40:40,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:40:40,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991240064, max timestamp=1731991240088Num uniques 859; 2024-11-19T04:40:40,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:40:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991300156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991300151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991300156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991300166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991300170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T04:40:40,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991300288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991300289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991300298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991300299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991300300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,352 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/309af6ddd91c4e03981e858628e36dd1 2024-11-19T04:40:40,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/475ca57c22ed4005a84b2949fcb5fcc9 is 50, key is test_row_0/B:col10/1731991239666/Put/seqid=0 2024-11-19T04:40:40,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741844_1020 (size=12001) 2024-11-19T04:40:40,395 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/475ca57c22ed4005a84b2949fcb5fcc9 2024-11-19T04:40:40,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/0f4514657094434a908533e4085f9f56 is 50, key is test_row_0/C:col10/1731991239666/Put/seqid=0 2024-11-19T04:40:40,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741845_1021 (size=12001) 2024-11-19T04:40:40,439 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/0f4514657094434a908533e4085f9f56 2024-11-19T04:40:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/309af6ddd91c4e03981e858628e36dd1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/309af6ddd91c4e03981e858628e36dd1 2024-11-19T04:40:40,470 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/309af6ddd91c4e03981e858628e36dd1, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T04:40:40,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/475ca57c22ed4005a84b2949fcb5fcc9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/475ca57c22ed4005a84b2949fcb5fcc9 2024-11-19T04:40:40,491 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/475ca57c22ed4005a84b2949fcb5fcc9, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T04:40:40,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/0f4514657094434a908533e4085f9f56 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/0f4514657094434a908533e4085f9f56 2024-11-19T04:40:40,504 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/0f4514657094434a908533e4085f9f56, entries=150, sequenceid=46, filesize=11.7 K 2024-11-19T04:40:40,506 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e97938390deb1164b875a27699276aad in 587ms, sequenceid=46, compaction requested=true 2024-11-19T04:40:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:40,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:40,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T04:40:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T04:40:40,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T04:40:40,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-11-19T04:40:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:40,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T04:40:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:40,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4660 sec 2024-11-19T04:40:40,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991300541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991300546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991300546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991300557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991300560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e3bddb51aece44f7b21af1fc10c67d99 is 50, key is test_row_0/A:col10/1731991240165/Put/seqid=0 2024-11-19T04:40:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741846_1022 (size=14341) 2024-11-19T04:40:40,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e3bddb51aece44f7b21af1fc10c67d99 2024-11-19T04:40:40,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/1a4230fc2bb84e0fa14751c08d4b694a is 50, key is test_row_0/B:col10/1731991240165/Put/seqid=0 2024-11-19T04:40:40,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991300668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991300669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991300669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991300677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991300678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741847_1023 (size=12001) 2024-11-19T04:40:40,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/1a4230fc2bb84e0fa14751c08d4b694a 2024-11-19T04:40:40,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/66a8cfbb42ef475f94e582dc6226240b is 50, key is test_row_0/C:col10/1731991240165/Put/seqid=0 2024-11-19T04:40:40,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741848_1024 (size=12001) 2024-11-19T04:40:40,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/66a8cfbb42ef475f94e582dc6226240b 2024-11-19T04:40:40,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e3bddb51aece44f7b21af1fc10c67d99 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e3bddb51aece44f7b21af1fc10c67d99 2024-11-19T04:40:40,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e3bddb51aece44f7b21af1fc10c67d99, entries=200, sequenceid=83, filesize=14.0 K 2024-11-19T04:40:40,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/1a4230fc2bb84e0fa14751c08d4b694a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1a4230fc2bb84e0fa14751c08d4b694a 2024-11-19T04:40:40,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991300888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991300889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991300893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:40,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1a4230fc2bb84e0fa14751c08d4b694a, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T04:40:40,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/66a8cfbb42ef475f94e582dc6226240b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/66a8cfbb42ef475f94e582dc6226240b 2024-11-19T04:40:40,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/66a8cfbb42ef475f94e582dc6226240b, entries=150, sequenceid=83, filesize=11.7 K 2024-11-19T04:40:40,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e97938390deb1164b875a27699276aad in 417ms, sequenceid=83, compaction requested=true 2024-11-19T04:40:40,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:40,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:40,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:40,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:40,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:40,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:40,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:40,947 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:40,947 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:40,952 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:40,952 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:40,954 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:40,954 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:40,954 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:40,954 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:40,955 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2fea7607fa0c4c88bec9112863f18d11, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3da1fc79b8045b691c76b36eda77dfb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/309af6ddd91c4e03981e858628e36dd1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e3bddb51aece44f7b21af1fc10c67d99] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=51.4 K 2024-11-19T04:40:40,955 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e2e740689cef47d1b036711ccfd76bba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b64b5d3bc52a41b9888704e3f9d01927, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/475ca57c22ed4005a84b2949fcb5fcc9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1a4230fc2bb84e0fa14751c08d4b694a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=46.9 K 2024-11-19T04:40:40,958 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e740689cef47d1b036711ccfd76bba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731991239100 2024-11-19T04:40:40,958 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fea7607fa0c4c88bec9112863f18d11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731991239100 2024-11-19T04:40:40,959 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3da1fc79b8045b691c76b36eda77dfb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731991239198 2024-11-19T04:40:40,959 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b64b5d3bc52a41b9888704e3f9d01927, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731991239198 2024-11-19T04:40:40,960 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 309af6ddd91c4e03981e858628e36dd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731991239666 2024-11-19T04:40:40,960 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 475ca57c22ed4005a84b2949fcb5fcc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731991239666 2024-11-19T04:40:40,962 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4230fc2bb84e0fa14751c08d4b694a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731991240162 2024-11-19T04:40:40,962 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3bddb51aece44f7b21af1fc10c67d99, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731991240144 2024-11-19T04:40:41,014 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#13 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:41,014 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#12 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:41,016 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/c2b6cd4d8423443c87252f09c6dbc0f8 is 50, key is test_row_0/B:col10/1731991240165/Put/seqid=0 2024-11-19T04:40:41,018 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/77c4870f2a6d483a90bfaf95912b1973 is 50, key is test_row_0/A:col10/1731991240165/Put/seqid=0 2024-11-19T04:40:41,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741850_1026 (size=12139) 2024-11-19T04:40:41,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741849_1025 (size=12139) 2024-11-19T04:40:41,067 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/77c4870f2a6d483a90bfaf95912b1973 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/77c4870f2a6d483a90bfaf95912b1973 2024-11-19T04:40:41,069 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T04:40:41,088 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into 77c4870f2a6d483a90bfaf95912b1973(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:41,088 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:41,092 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=12, startTime=1731991240937; duration=0sec 2024-11-19T04:40:41,093 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:41,093 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:41,093 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:41,099 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:41,099 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:41,099 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,099 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b477cc5a8578479ca3ddf7999957fd6a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e07d76691512480384ceee54dc78cf2b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/0f4514657094434a908533e4085f9f56, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/66a8cfbb42ef475f94e582dc6226240b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=46.9 K 2024-11-19T04:40:41,104 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b477cc5a8578479ca3ddf7999957fd6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731991239100 2024-11-19T04:40:41,105 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e07d76691512480384ceee54dc78cf2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731991239198 2024-11-19T04:40:41,106 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f4514657094434a908533e4085f9f56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731991239666 2024-11-19T04:40:41,106 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a8cfbb42ef475f94e582dc6226240b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731991240162 2024-11-19T04:40:41,152 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:41,153 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/64acfff3ef464d5f947a5398eac893c3 is 50, key is test_row_0/C:col10/1731991240165/Put/seqid=0 2024-11-19T04:40:41,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741851_1027 (size=12139) 2024-11-19T04:40:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T04:40:41,208 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:41,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:41,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:40:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T04:40:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:41,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T04:40:41,212 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:41,215 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:41,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:41,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e is 50, key is test_row_0/A:col10/1731991241207/Put/seqid=0 2024-11-19T04:40:41,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741852_1028 (size=12001) 2024-11-19T04:40:41,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991301270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991301271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991301275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991301277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991301279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T04:40:41,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991301387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991301388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991301388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991301398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991301398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,464 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/c2b6cd4d8423443c87252f09c6dbc0f8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/c2b6cd4d8423443c87252f09c6dbc0f8 2024-11-19T04:40:41,478 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into c2b6cd4d8423443c87252f09c6dbc0f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:41,478 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:41,478 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=12, startTime=1731991240944; duration=0sec 2024-11-19T04:40:41,479 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:41,479 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T04:40:41,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,587 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/64acfff3ef464d5f947a5398eac893c3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/64acfff3ef464d5f947a5398eac893c3 2024-11-19T04:40:41,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991301607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,613 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into 64acfff3ef464d5f947a5398eac893c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:41,613 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:41,613 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=12, startTime=1731991240945; duration=0sec 2024-11-19T04:40:41,614 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,614 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991301608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991301608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991301608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991301609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e 2024-11-19T04:40:41,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/95705875648a493db45b1f190b272b7e is 50, key is test_row_0/B:col10/1731991241207/Put/seqid=0 2024-11-19T04:40:41,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741853_1029 (size=12001) 2024-11-19T04:40:41,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/95705875648a493db45b1f190b272b7e 2024-11-19T04:40:41,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/e530d8a00f9c44cf9f470b323cd4dea3 is 50, key is test_row_0/C:col10/1731991241207/Put/seqid=0 2024-11-19T04:40:41,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741854_1030 (size=12001) 2024-11-19T04:40:41,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T04:40:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T04:40:41,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:41,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:41,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:41,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991301918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991301927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991301928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991301928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:41,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991301928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:42,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:42,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:42,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:42,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/e530d8a00f9c44cf9f470b323cd4dea3 2024-11-19T04:40:42,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e 2024-11-19T04:40:42,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T04:40:42,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/95705875648a493db45b1f190b272b7e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/95705875648a493db45b1f190b272b7e 2024-11-19T04:40:42,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/95705875648a493db45b1f190b272b7e, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T04:40:42,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/e530d8a00f9c44cf9f470b323cd4dea3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e530d8a00f9c44cf9f470b323cd4dea3 2024-11-19T04:40:42,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e530d8a00f9c44cf9f470b323cd4dea3, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T04:40:42,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e97938390deb1164b875a27699276aad in 1071ms, sequenceid=97, compaction requested=false 2024-11-19T04:40:42,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:42,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T04:40:42,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:42,313 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T04:40:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:42,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8650e133f1a94283a6eeae2b75ba7061 is 50, key is test_row_0/A:col10/1731991241275/Put/seqid=0 2024-11-19T04:40:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T04:40:42,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741855_1031 (size=12001) 2024-11-19T04:40:42,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991302467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991302468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991302473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991302477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991302477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991302588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991302588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991302591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991302598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991302598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,658 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T04:40:42,659 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T04:40:42,661 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T04:40:42,661 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T04:40:42,661 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T04:40:42,661 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T04:40:42,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T04:40:42,663 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T04:40:42,794 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8650e133f1a94283a6eeae2b75ba7061 2024-11-19T04:40:42,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991302798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991302798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991302808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991302808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991302809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/9072f45211564756bb29892d65880a02 is 50, key is test_row_0/B:col10/1731991241275/Put/seqid=0 2024-11-19T04:40:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:42,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741856_1032 (size=12001) 2024-11-19T04:40:42,850 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/9072f45211564756bb29892d65880a02 2024-11-19T04:40:42,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9361f5513c124a1fbebe0535cfbfc0fb is 50, key is test_row_0/C:col10/1731991241275/Put/seqid=0 2024-11-19T04:40:42,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741857_1033 (size=12001) 2024-11-19T04:40:42,933 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9361f5513c124a1fbebe0535cfbfc0fb 2024-11-19T04:40:42,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8650e133f1a94283a6eeae2b75ba7061 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8650e133f1a94283a6eeae2b75ba7061 2024-11-19T04:40:42,963 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8650e133f1a94283a6eeae2b75ba7061, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T04:40:42,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/9072f45211564756bb29892d65880a02 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/9072f45211564756bb29892d65880a02 2024-11-19T04:40:42,988 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/9072f45211564756bb29892d65880a02, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T04:40:42,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9361f5513c124a1fbebe0535cfbfc0fb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9361f5513c124a1fbebe0535cfbfc0fb 2024-11-19T04:40:43,009 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9361f5513c124a1fbebe0535cfbfc0fb, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T04:40:43,012 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e97938390deb1164b875a27699276aad in 698ms, sequenceid=118, compaction requested=true 2024-11-19T04:40:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T04:40:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T04:40:43,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T04:40:43,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-19T04:40:43,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8130 sec 2024-11-19T04:40:43,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T04:40:43,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:43,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/b510b14d914e47efbce5958b5f2f787b is 50, key is test_row_0/A:col10/1731991242464/Put/seqid=0 2024-11-19T04:40:43,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741858_1034 (size=14491) 2024-11-19T04:40:43,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/b510b14d914e47efbce5958b5f2f787b 2024-11-19T04:40:43,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b9a003b3280a480daf965285d4472d1a is 50, key is test_row_0/B:col10/1731991242464/Put/seqid=0 2024-11-19T04:40:43,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991303166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991303166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991303167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991303170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991303176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741859_1035 (size=12101) 2024-11-19T04:40:43,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b9a003b3280a480daf965285d4472d1a 2024-11-19T04:40:43,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/7ed9c2ad30df41afa17ceab33190a916 is 50, key is test_row_0/C:col10/1731991242464/Put/seqid=0 2024-11-19T04:40:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741860_1036 (size=12101) 2024-11-19T04:40:43,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/7ed9c2ad30df41afa17ceab33190a916 2024-11-19T04:40:43,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/b510b14d914e47efbce5958b5f2f787b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/b510b14d914e47efbce5958b5f2f787b 2024-11-19T04:40:43,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/b510b14d914e47efbce5958b5f2f787b, entries=200, sequenceid=133, filesize=14.2 K 2024-11-19T04:40:43,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b9a003b3280a480daf965285d4472d1a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b9a003b3280a480daf965285d4472d1a 2024-11-19T04:40:43,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b9a003b3280a480daf965285d4472d1a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T04:40:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/7ed9c2ad30df41afa17ceab33190a916 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/7ed9c2ad30df41afa17ceab33190a916 2024-11-19T04:40:43,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/7ed9c2ad30df41afa17ceab33190a916, entries=150, sequenceid=133, filesize=11.8 K 2024-11-19T04:40:43,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e97938390deb1164b875a27699276aad in 172ms, sequenceid=133, compaction requested=true 2024-11-19T04:40:43,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:43,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:43,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:43,297 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:43,297 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:43,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:43,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:43,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:43,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T04:40:43,301 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:43,301 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:43,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:43,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:43,302 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,303 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/77c4870f2a6d483a90bfaf95912b1973, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8650e133f1a94283a6eeae2b75ba7061, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/b510b14d914e47efbce5958b5f2f787b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=49.4 K 2024-11-19T04:40:43,303 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,303 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/c2b6cd4d8423443c87252f09c6dbc0f8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/95705875648a493db45b1f190b272b7e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/9072f45211564756bb29892d65880a02, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b9a003b3280a480daf965285d4472d1a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=47.1 K 2024-11-19T04:40:43,304 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c4870f2a6d483a90bfaf95912b1973, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731991240162 2024-11-19T04:40:43,304 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b6cd4d8423443c87252f09c6dbc0f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731991240162 2024-11-19T04:40:43,305 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1e7f631c7243e2a1c3b6f54ffc0b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731991240530 2024-11-19T04:40:43,305 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 95705875648a493db45b1f190b272b7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731991240530 2024-11-19T04:40:43,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:43,306 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8650e133f1a94283a6eeae2b75ba7061, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731991241262 2024-11-19T04:40:43,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:43,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:43,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,307 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9072f45211564756bb29892d65880a02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731991241262 2024-11-19T04:40:43,307 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b510b14d914e47efbce5958b5f2f787b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731991242464 2024-11-19T04:40:43,308 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a003b3280a480daf965285d4472d1a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731991242464 2024-11-19T04:40:43,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/71c1b916f95d49c3a1e6053327896245 is 50, key is test_row_0/A:col10/1731991243298/Put/seqid=0 2024-11-19T04:40:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991303330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991303332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991303335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991303339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,352 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#26 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991303345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,353 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#25 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,354 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/6dc9301006364d3d9c7b9b8c35b6dbca is 50, key is test_row_0/B:col10/1731991242464/Put/seqid=0 2024-11-19T04:40:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T04:40:43,357 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:43,357 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/9fc515991a734c8697b2f3347e2287e9 is 50, key is test_row_0/A:col10/1731991242464/Put/seqid=0 2024-11-19T04:40:43,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T04:40:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T04:40:43,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:43,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741861_1037 (size=16931) 2024-11-19T04:40:43,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/71c1b916f95d49c3a1e6053327896245 2024-11-19T04:40:43,363 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:43,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:43,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741862_1038 (size=12375) 2024-11-19T04:40:43,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741863_1039 (size=12375) 2024-11-19T04:40:43,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/33f063389a3b48a5be036725f17bab71 is 50, key is test_row_0/B:col10/1731991243298/Put/seqid=0 2024-11-19T04:40:43,413 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/9fc515991a734c8697b2f3347e2287e9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/9fc515991a734c8697b2f3347e2287e9 2024-11-19T04:40:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741864_1040 (size=12151) 2024-11-19T04:40:43,426 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into 9fc515991a734c8697b2f3347e2287e9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:43,426 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:43,426 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=12, startTime=1731991243297; duration=0sec 2024-11-19T04:40:43,427 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:43,427 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:43,427 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:43,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/33f063389a3b48a5be036725f17bab71 2024-11-19T04:40:43,430 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:43,430 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:43,430 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,430 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/64acfff3ef464d5f947a5398eac893c3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e530d8a00f9c44cf9f470b323cd4dea3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9361f5513c124a1fbebe0535cfbfc0fb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/7ed9c2ad30df41afa17ceab33190a916] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=47.1 K 2024-11-19T04:40:43,432 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64acfff3ef464d5f947a5398eac893c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731991240162 2024-11-19T04:40:43,433 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e530d8a00f9c44cf9f470b323cd4dea3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731991240530 2024-11-19T04:40:43,437 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9361f5513c124a1fbebe0535cfbfc0fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731991241262 2024-11-19T04:40:43,439 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed9c2ad30df41afa17ceab33190a916, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731991242464 2024-11-19T04:40:43,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/49bfee3dd72d464abb8ddaf47831a17b is 50, key is test_row_0/C:col10/1731991243298/Put/seqid=0 2024-11-19T04:40:43,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991303447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T04:40:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991303447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991303472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991303474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991303474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,500 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:43,501 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/d081b4065af24e7e9da5a82c441148a8 is 50, key is test_row_0/C:col10/1731991242464/Put/seqid=0 2024-11-19T04:40:43,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T04:40:43,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:43,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741865_1041 (size=12151) 2024-11-19T04:40:43,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741866_1042 (size=12375) 2024-11-19T04:40:43,563 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/d081b4065af24e7e9da5a82c441148a8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/d081b4065af24e7e9da5a82c441148a8 2024-11-19T04:40:43,574 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into d081b4065af24e7e9da5a82c441148a8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:43,574 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:43,574 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=12, startTime=1731991243298; duration=0sec 2024-11-19T04:40:43,575 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:43,575 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:43,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T04:40:43,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:43,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T04:40:43,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991303678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991303688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991303688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991303688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991303688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:43,809 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/6dc9301006364d3d9c7b9b8c35b6dbca as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/6dc9301006364d3d9c7b9b8c35b6dbca 2024-11-19T04:40:43,819 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into 6dc9301006364d3d9c7b9b8c35b6dbca(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:43,819 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:43,819 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=12, startTime=1731991243297; duration=0sec 2024-11-19T04:40:43,819 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:43,819 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:43,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T04:40:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:43,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:43,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/49bfee3dd72d464abb8ddaf47831a17b 2024-11-19T04:40:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/71c1b916f95d49c3a1e6053327896245 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/71c1b916f95d49c3a1e6053327896245 2024-11-19T04:40:43,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/71c1b916f95d49c3a1e6053327896245, entries=250, sequenceid=152, filesize=16.5 K 2024-11-19T04:40:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/33f063389a3b48a5be036725f17bab71 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/33f063389a3b48a5be036725f17bab71 2024-11-19T04:40:43,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/33f063389a3b48a5be036725f17bab71, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T04:40:43,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/49bfee3dd72d464abb8ddaf47831a17b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/49bfee3dd72d464abb8ddaf47831a17b 2024-11-19T04:40:43,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/49bfee3dd72d464abb8ddaf47831a17b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-19T04:40:43,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e97938390deb1164b875a27699276aad in 683ms, sequenceid=152, compaction requested=false 2024-11-19T04:40:43,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T04:40:43,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T04:40:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:43,989 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:40:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:43,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:43,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e0b3402b677f420b950daacd4d94420a is 50, key is test_row_0/A:col10/1731991243988/Put/seqid=0 2024-11-19T04:40:44,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741867_1043 (size=14541) 2024-11-19T04:40:44,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991304027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991304031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991304033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991304033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991304042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991304147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991304148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991304148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991304148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991304157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991304357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991304357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991304357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991304358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991304367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,408 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e0b3402b677f420b950daacd4d94420a 2024-11-19T04:40:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/a944a5cf8b8f4e7b87fad8eeff82ea4c is 50, key is test_row_0/B:col10/1731991243988/Put/seqid=0 2024-11-19T04:40:44,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741868_1044 (size=12151) 2024-11-19T04:40:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T04:40:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991304667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991304667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991304667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991304667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991304677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:44,840 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/a944a5cf8b8f4e7b87fad8eeff82ea4c 2024-11-19T04:40:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/725a422c7e2540d883a14ed3e6ee18e6 is 50, key is test_row_0/C:col10/1731991243988/Put/seqid=0 2024-11-19T04:40:44,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741869_1045 (size=12151) 2024-11-19T04:40:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991305177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991305177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:45,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:45,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991305187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991305188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991305197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:45,277 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/725a422c7e2540d883a14ed3e6ee18e6 2024-11-19T04:40:45,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e0b3402b677f420b950daacd4d94420a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e0b3402b677f420b950daacd4d94420a 2024-11-19T04:40:45,297 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e0b3402b677f420b950daacd4d94420a, entries=200, sequenceid=169, filesize=14.2 K 2024-11-19T04:40:45,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/a944a5cf8b8f4e7b87fad8eeff82ea4c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a944a5cf8b8f4e7b87fad8eeff82ea4c 2024-11-19T04:40:45,308 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a944a5cf8b8f4e7b87fad8eeff82ea4c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T04:40:45,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/725a422c7e2540d883a14ed3e6ee18e6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/725a422c7e2540d883a14ed3e6ee18e6 2024-11-19T04:40:45,321 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/725a422c7e2540d883a14ed3e6ee18e6, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T04:40:45,324 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e97938390deb1164b875a27699276aad in 1334ms, sequenceid=169, compaction requested=true 2024-11-19T04:40:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T04:40:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T04:40:45,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T04:40:45,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9620 sec 2024-11-19T04:40:45,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9760 sec 2024-11-19T04:40:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T04:40:45,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:45,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T04:40:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T04:40:45,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:45,513 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:45,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T04:40:45,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T04:40:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:45,666 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T04:40:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:45,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8887c93f441b426e8ff06b97d6a4a8bc is 50, key is test_row_0/A:col10/1731991244037/Put/seqid=0 2024-11-19T04:40:45,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741870_1046 (size=12151) 2024-11-19T04:40:45,683 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8887c93f441b426e8ff06b97d6a4a8bc 2024-11-19T04:40:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/ebe315594c0f44009d1c9c1c91982fa7 is 50, key is test_row_0/B:col10/1731991244037/Put/seqid=0 2024-11-19T04:40:45,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741871_1047 (size=12151) 2024-11-19T04:40:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T04:40:46,109 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/ebe315594c0f44009d1c9c1c91982fa7 2024-11-19T04:40:46,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/c398227c260847e4a1eac23edf36ceef is 50, key is test_row_0/C:col10/1731991244037/Put/seqid=0 2024-11-19T04:40:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T04:40:46,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741872_1048 (size=12151) 2024-11-19T04:40:46,154 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/c398227c260847e4a1eac23edf36ceef 2024-11-19T04:40:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/8887c93f441b426e8ff06b97d6a4a8bc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8887c93f441b426e8ff06b97d6a4a8bc 2024-11-19T04:40:46,176 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8887c93f441b426e8ff06b97d6a4a8bc, entries=150, sequenceid=187, filesize=11.9 K 2024-11-19T04:40:46,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/ebe315594c0f44009d1c9c1c91982fa7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ebe315594c0f44009d1c9c1c91982fa7 2024-11-19T04:40:46,186 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ebe315594c0f44009d1c9c1c91982fa7, entries=150, sequenceid=187, filesize=11.9 K 2024-11-19T04:40:46,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/c398227c260847e4a1eac23edf36ceef as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c398227c260847e4a1eac23edf36ceef 2024-11-19T04:40:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:46,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:46,200 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c398227c260847e4a1eac23edf36ceef, entries=150, sequenceid=187, filesize=11.9 K 2024-11-19T04:40:46,202 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for e97938390deb1164b875a27699276aad in 535ms, sequenceid=187, compaction requested=true 2024-11-19T04:40:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:46,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T04:40:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T04:40:46,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T04:40:46,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 691 msec 2024-11-19T04:40:46,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 702 msec 2024-11-19T04:40:46,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:40:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:46,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:46,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c67d86886f254fd2970391712935b59f is 50, key is test_row_0/A:col10/1731991246209/Put/seqid=0 2024-11-19T04:40:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741873_1049 (size=14541) 2024-11-19T04:40:46,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c67d86886f254fd2970391712935b59f 2024-11-19T04:40:46,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/11a9059129a24d55acb818a5fe48b5a6 is 50, key is test_row_0/B:col10/1731991246209/Put/seqid=0 2024-11-19T04:40:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741874_1050 (size=12151) 2024-11-19T04:40:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991246302, max timestamp=1731991246323Num uniques 859; 2024-11-19T04:40:46,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:40:46,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731991246318, max timestamp=1731991246323Num uniques 859; 2024-11-19T04:40:46,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T04:40:46,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731991246318, max timestamp=1731991246323Num uniques 859; 2024-11-19T04:40:46,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T04:40:46,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991306376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991306376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991306379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991306388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991306390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991306497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991306497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991306497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991306508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991306508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T04:40:46,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:46,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:46,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T04:40:46,652 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T04:40:46,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:46,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:46,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/11a9059129a24d55acb818a5fe48b5a6 2024-11-19T04:40:46,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f3ac4e0a337e49a8820b6975f45e04ae is 50, key is test_row_0/C:col10/1731991246209/Put/seqid=0 2024-11-19T04:40:46,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741875_1051 (size=12151) 2024-11-19T04:40:46,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f3ac4e0a337e49a8820b6975f45e04ae 2024-11-19T04:40:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c67d86886f254fd2970391712935b59f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c67d86886f254fd2970391712935b59f 2024-11-19T04:40:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991306708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991306708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991306708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991306717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991306717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:46,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:46,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c67d86886f254fd2970391712935b59f, entries=200, sequenceid=198, filesize=14.2 K 2024-11-19T04:40:46,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/11a9059129a24d55acb818a5fe48b5a6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/11a9059129a24d55acb818a5fe48b5a6 2024-11-19T04:40:46,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/11a9059129a24d55acb818a5fe48b5a6, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T04:40:46,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f3ac4e0a337e49a8820b6975f45e04ae as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f3ac4e0a337e49a8820b6975f45e04ae 2024-11-19T04:40:46,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f3ac4e0a337e49a8820b6975f45e04ae, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T04:40:46,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=135.63 KB/138880 for e97938390deb1164b875a27699276aad in 533ms, sequenceid=198, compaction requested=true 2024-11-19T04:40:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:46,748 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:40:46,748 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:40:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:46,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:46,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:46,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:46,752 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 70539 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:40:46,753 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:46,753 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:46,753 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/9fc515991a734c8697b2f3347e2287e9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/71c1b916f95d49c3a1e6053327896245, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e0b3402b677f420b950daacd4d94420a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8887c93f441b426e8ff06b97d6a4a8bc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c67d86886f254fd2970391712935b59f] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=68.9 K 2024-11-19T04:40:46,753 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60979 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:40:46,753 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:46,754 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:46,754 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/6dc9301006364d3d9c7b9b8c35b6dbca, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/33f063389a3b48a5be036725f17bab71, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a944a5cf8b8f4e7b87fad8eeff82ea4c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ebe315594c0f44009d1c9c1c91982fa7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/11a9059129a24d55acb818a5fe48b5a6] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=59.5 K 2024-11-19T04:40:46,754 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc515991a734c8697b2f3347e2287e9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731991242464 2024-11-19T04:40:46,754 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc9301006364d3d9c7b9b8c35b6dbca, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731991242464 2024-11-19T04:40:46,755 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c1b916f95d49c3a1e6053327896245, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731991243148 2024-11-19T04:40:46,755 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 33f063389a3b48a5be036725f17bab71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731991243151 2024-11-19T04:40:46,755 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0b3402b677f420b950daacd4d94420a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731991243320 2024-11-19T04:40:46,755 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a944a5cf8b8f4e7b87fad8eeff82ea4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731991243332 2024-11-19T04:40:46,756 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8887c93f441b426e8ff06b97d6a4a8bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731991244025 2024-11-19T04:40:46,756 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe315594c0f44009d1c9c1c91982fa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731991244025 2024-11-19T04:40:46,757 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a9059129a24d55acb818a5fe48b5a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991246209 2024-11-19T04:40:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T04:40:46,757 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c67d86886f254fd2970391712935b59f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991246197 2024-11-19T04:40:46,784 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:46,784 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:46,785 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/ab1714882de14389bbd34cb6ccb9df61 is 50, key is test_row_0/A:col10/1731991246209/Put/seqid=0 2024-11-19T04:40:46,785 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/fb6698c2d4964cf0954612a36e6e3689 is 50, key is test_row_0/B:col10/1731991246209/Put/seqid=0 2024-11-19T04:40:46,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741876_1052 (size=12595) 2024-11-19T04:40:46,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741877_1053 (size=12595) 2024-11-19T04:40:46,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T04:40:46,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:46,812 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-11-19T04:40:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:46,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c3de997bfded428ab235bbb0960e2889 is 50, key is test_row_0/A:col10/1731991246387/Put/seqid=0 2024-11-19T04:40:46,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741878_1054 (size=12151) 2024-11-19T04:40:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T04:40:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:47,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991307037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991307037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991307038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991307040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991307052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991307176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991307176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991307176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991307176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991307177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,202 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/ab1714882de14389bbd34cb6ccb9df61 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ab1714882de14389bbd34cb6ccb9df61 2024-11-19T04:40:47,208 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/fb6698c2d4964cf0954612a36e6e3689 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fb6698c2d4964cf0954612a36e6e3689 2024-11-19T04:40:47,212 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into ab1714882de14389bbd34cb6ccb9df61(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:47,213 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:47,213 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=11, startTime=1731991246748; duration=0sec 2024-11-19T04:40:47,213 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:47,213 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:47,213 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:40:47,215 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60979 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:40:47,215 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:47,215 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:47,215 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/d081b4065af24e7e9da5a82c441148a8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/49bfee3dd72d464abb8ddaf47831a17b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/725a422c7e2540d883a14ed3e6ee18e6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c398227c260847e4a1eac23edf36ceef, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f3ac4e0a337e49a8820b6975f45e04ae] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=59.5 K 2024-11-19T04:40:47,216 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d081b4065af24e7e9da5a82c441148a8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731991242464 2024-11-19T04:40:47,217 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49bfee3dd72d464abb8ddaf47831a17b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731991243151 2024-11-19T04:40:47,218 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725a422c7e2540d883a14ed3e6ee18e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731991243332 2024-11-19T04:40:47,218 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into fb6698c2d4964cf0954612a36e6e3689(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:47,218 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c398227c260847e4a1eac23edf36ceef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731991244025 2024-11-19T04:40:47,218 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:47,218 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=11, startTime=1731991246748; duration=0sec 2024-11-19T04:40:47,218 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:47,219 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:47,219 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ac4e0a337e49a8820b6975f45e04ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991246209 2024-11-19T04:40:47,235 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:47,235 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/cc0a759d0d3047bbabd89820f998c671 is 50, key is test_row_0/C:col10/1731991246209/Put/seqid=0 2024-11-19T04:40:47,236 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c3de997bfded428ab235bbb0960e2889 2024-11-19T04:40:47,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741879_1055 (size=12595) 2024-11-19T04:40:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/60dc98ffb50d416e816f68f64c92e7f5 is 50, key is test_row_0/B:col10/1731991246387/Put/seqid=0 2024-11-19T04:40:47,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741880_1056 (size=12151) 2024-11-19T04:40:47,265 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/60dc98ffb50d416e816f68f64c92e7f5 2024-11-19T04:40:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9b430b0137554065965c436bc1928062 is 50, key is test_row_0/C:col10/1731991246387/Put/seqid=0 2024-11-19T04:40:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T04:40:47,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741881_1057 (size=12151) 2024-11-19T04:40:47,292 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9b430b0137554065965c436bc1928062 2024-11-19T04:40:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/c3de997bfded428ab235bbb0960e2889 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3de997bfded428ab235bbb0960e2889 2024-11-19T04:40:47,311 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3de997bfded428ab235bbb0960e2889, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T04:40:47,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/60dc98ffb50d416e816f68f64c92e7f5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/60dc98ffb50d416e816f68f64c92e7f5 2024-11-19T04:40:47,324 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/60dc98ffb50d416e816f68f64c92e7f5, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T04:40:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9b430b0137554065965c436bc1928062 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9b430b0137554065965c436bc1928062 2024-11-19T04:40:47,337 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9b430b0137554065965c436bc1928062, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T04:40:47,339 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138880, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for e97938390deb1164b875a27699276aad in 526ms, sequenceid=235, compaction requested=false 2024-11-19T04:40:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T04:40:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T04:40:47,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T04:40:47,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 690 msec 2024-11-19T04:40:47,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 700 msec 2024-11-19T04:40:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:47,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:40:47,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:47,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:47,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:47,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:47,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:47,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:47,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/6d34c8e48e8047c2a1fb926805d4259d is 50, key is test_row_0/A:col10/1731991247390/Put/seqid=0 2024-11-19T04:40:47,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741882_1058 (size=12151) 2024-11-19T04:40:47,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991307419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991307420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991307422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991307425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991307431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991307537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991307537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991307537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991307547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991307547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,658 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/cc0a759d0d3047bbabd89820f998c671 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/cc0a759d0d3047bbabd89820f998c671 2024-11-19T04:40:47,667 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into cc0a759d0d3047bbabd89820f998c671(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:47,667 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:47,667 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=11, startTime=1731991246749; duration=0sec 2024-11-19T04:40:47,668 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:47,668 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:47,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991307748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991307748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991307748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991307757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991307757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T04:40:47,787 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:47,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T04:40:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T04:40:47,791 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:47,792 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:47,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:47,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/6d34c8e48e8047c2a1fb926805d4259d 2024-11-19T04:40:47,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/2c9bfc72afbf47d7ad66d684c234e391 is 50, key is test_row_0/B:col10/1731991247390/Put/seqid=0 2024-11-19T04:40:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741883_1059 (size=12151) 2024-11-19T04:40:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T04:40:47,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T04:40:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:47,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:47,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991308057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991308057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991308057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991308067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991308077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T04:40:48,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:48,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T04:40:48,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/2c9bfc72afbf47d7ad66d684c234e391 2024-11-19T04:40:48,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/28d6fe25f2984fca85dc04c12d385d1f is 50, key is test_row_0/C:col10/1731991247390/Put/seqid=0 2024-11-19T04:40:48,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741884_1060 (size=12151) 2024-11-19T04:40:48,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T04:40:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:48,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T04:40:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T04:40:48,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T04:40:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:48,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991308567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991308568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991308568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991308577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991308587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:48,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/28d6fe25f2984fca85dc04c12d385d1f 2024-11-19T04:40:48,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/6d34c8e48e8047c2a1fb926805d4259d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6d34c8e48e8047c2a1fb926805d4259d 2024-11-19T04:40:48,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6d34c8e48e8047c2a1fb926805d4259d, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T04:40:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/2c9bfc72afbf47d7ad66d684c234e391 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/2c9bfc72afbf47d7ad66d684c234e391 2024-11-19T04:40:48,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/2c9bfc72afbf47d7ad66d684c234e391, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T04:40:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/28d6fe25f2984fca85dc04c12d385d1f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/28d6fe25f2984fca85dc04c12d385d1f 2024-11-19T04:40:48,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/28d6fe25f2984fca85dc04c12d385d1f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-19T04:40:48,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e97938390deb1164b875a27699276aad in 1299ms, sequenceid=249, compaction requested=true 2024-11-19T04:40:48,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:48,690 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:48,690 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:48,691 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:48,691 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:48,691 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:48,691 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,691 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:48,691 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,691 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ab1714882de14389bbd34cb6ccb9df61, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3de997bfded428ab235bbb0960e2889, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6d34c8e48e8047c2a1fb926805d4259d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.0 K 2024-11-19T04:40:48,691 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fb6698c2d4964cf0954612a36e6e3689, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/60dc98ffb50d416e816f68f64c92e7f5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/2c9bfc72afbf47d7ad66d684c234e391] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.0 K 2024-11-19T04:40:48,692 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab1714882de14389bbd34cb6ccb9df61, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991246209 2024-11-19T04:40:48,692 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6698c2d4964cf0954612a36e6e3689, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991246209 2024-11-19T04:40:48,693 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 60dc98ffb50d416e816f68f64c92e7f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731991246370 2024-11-19T04:40:48,693 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3de997bfded428ab235bbb0960e2889, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731991246370 2024-11-19T04:40:48,693 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9bfc72afbf47d7ad66d684c234e391, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731991247034 2024-11-19T04:40:48,693 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d34c8e48e8047c2a1fb926805d4259d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731991247034 2024-11-19T04:40:48,705 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:48,706 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/fdb9f2e30e8d4f87a0e5054632001eb9 is 50, key is test_row_0/B:col10/1731991247390/Put/seqid=0 2024-11-19T04:40:48,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T04:40:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:48,716 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:40:48,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:48,718 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:48,719 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/05ddd007bc3e4976aaed8a1aedfc4499 is 50, key is test_row_0/A:col10/1731991247390/Put/seqid=0 2024-11-19T04:40:48,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741885_1061 (size=12697) 2024-11-19T04:40:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/fd74a6f720a041e9890997294c59f1e8 is 50, key is test_row_0/A:col10/1731991247419/Put/seqid=0 2024-11-19T04:40:48,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741886_1062 (size=12697) 2024-11-19T04:40:48,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741887_1063 (size=12301) 2024-11-19T04:40:48,733 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/fd74a6f720a041e9890997294c59f1e8 2024-11-19T04:40:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e6ef3b16c4d54805aa7396d93ef706d8 is 50, key is test_row_0/B:col10/1731991247419/Put/seqid=0 2024-11-19T04:40:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741888_1064 (size=12301) 2024-11-19T04:40:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T04:40:49,134 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/fdb9f2e30e8d4f87a0e5054632001eb9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fdb9f2e30e8d4f87a0e5054632001eb9 2024-11-19T04:40:49,138 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/05ddd007bc3e4976aaed8a1aedfc4499 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/05ddd007bc3e4976aaed8a1aedfc4499 2024-11-19T04:40:49,145 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into fdb9f2e30e8d4f87a0e5054632001eb9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:49,145 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:49,145 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=13, startTime=1731991248690; duration=0sec 2024-11-19T04:40:49,145 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:49,145 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:49,145 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:49,147 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:49,147 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:49,147 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into 05ddd007bc3e4976aaed8a1aedfc4499(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:49,147 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:49,147 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:49,148 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=13, startTime=1731991248690; duration=0sec 2024-11-19T04:40:49,148 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/cc0a759d0d3047bbabd89820f998c671, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9b430b0137554065965c436bc1928062, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/28d6fe25f2984fca85dc04c12d385d1f] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.0 K 2024-11-19T04:40:49,148 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:49,148 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:49,148 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0a759d0d3047bbabd89820f998c671, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991246209 2024-11-19T04:40:49,149 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b430b0137554065965c436bc1928062, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731991246370 2024-11-19T04:40:49,149 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 28d6fe25f2984fca85dc04c12d385d1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731991247034 2024-11-19T04:40:49,160 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:49,161 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/8e7479ff306b4ae4b70f8ecb2f7a2439 is 50, key is test_row_0/C:col10/1731991247390/Put/seqid=0 2024-11-19T04:40:49,163 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e6ef3b16c4d54805aa7396d93ef706d8 2024-11-19T04:40:49,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741889_1065 (size=12697) 2024-11-19T04:40:49,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f6fb404d5d73438fae4346d11c2c288f is 50, key is test_row_0/C:col10/1731991247419/Put/seqid=0 2024-11-19T04:40:49,183 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/8e7479ff306b4ae4b70f8ecb2f7a2439 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8e7479ff306b4ae4b70f8ecb2f7a2439 2024-11-19T04:40:49,193 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into 8e7479ff306b4ae4b70f8ecb2f7a2439(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:49,193 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:49,193 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=13, startTime=1731991248690; duration=0sec 2024-11-19T04:40:49,193 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:49,193 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:49,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741890_1066 (size=12301) 2024-11-19T04:40:49,205 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f6fb404d5d73438fae4346d11c2c288f 2024-11-19T04:40:49,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/fd74a6f720a041e9890997294c59f1e8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/fd74a6f720a041e9890997294c59f1e8 2024-11-19T04:40:49,220 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/fd74a6f720a041e9890997294c59f1e8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T04:40:49,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e6ef3b16c4d54805aa7396d93ef706d8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e6ef3b16c4d54805aa7396d93ef706d8 2024-11-19T04:40:49,231 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e6ef3b16c4d54805aa7396d93ef706d8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T04:40:49,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f6fb404d5d73438fae4346d11c2c288f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f6fb404d5d73438fae4346d11c2c288f 2024-11-19T04:40:49,244 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f6fb404d5d73438fae4346d11c2c288f, entries=150, sequenceid=270, filesize=12.0 K 2024-11-19T04:40:49,245 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for e97938390deb1164b875a27699276aad in 529ms, sequenceid=270, compaction requested=false 2024-11-19T04:40:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:49,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T04:40:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T04:40:49,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T04:40:49,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-19T04:40:49,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4640 sec 2024-11-19T04:40:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:49,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:40:49,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:49,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:49,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:49,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:49,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:49,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:49,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/a8d3b13abd4b4f1da2f454449c363243 is 50, key is test_row_0/A:col10/1731991249584/Put/seqid=0 2024-11-19T04:40:49,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741891_1067 (size=12301) 2024-11-19T04:40:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991309645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991309643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991309649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991249634, max timestamp=1731991249654Num uniques 859; 2024-11-19T04:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991249634, max timestamp=1731991249654Num uniques 859; 2024-11-19T04:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:40:49,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991249634, max timestamp=1731991249653Num uniques 859; 2024-11-19T04:40:49,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:40:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991309721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991309722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991309767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991309767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991309767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991309827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991309837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T04:40:49,937 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:49,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T04:40:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T04:40:49,940 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:49,941 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:49,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:49,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991309987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991309987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991309988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/a8d3b13abd4b4f1da2f454449c363243 2024-11-19T04:40:50,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/ed6a5536454647d2868d5a69ff37a5f2 is 50, key is test_row_0/B:col10/1731991249584/Put/seqid=0 2024-11-19T04:40:50,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741892_1068 (size=12301) 2024-11-19T04:40:50,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991310037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T04:40:50,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991310047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T04:40:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T04:40:50,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T04:40:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991310297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991310298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991310298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991310346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991310357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T04:40:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/ed6a5536454647d2868d5a69ff37a5f2 2024-11-19T04:40:50,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9825540f42ae426c97f37c90dec5f424 is 50, key is test_row_0/C:col10/1731991249584/Put/seqid=0 2024-11-19T04:40:50,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741893_1069 (size=12301) 2024-11-19T04:40:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T04:40:50,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T04:40:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T04:40:50,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:50,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:50,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991310817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991310817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991310817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9825540f42ae426c97f37c90dec5f424 2024-11-19T04:40:50,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/a8d3b13abd4b4f1da2f454449c363243 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/a8d3b13abd4b4f1da2f454449c363243 2024-11-19T04:40:50,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/a8d3b13abd4b4f1da2f454449c363243, entries=150, sequenceid=283, filesize=12.0 K 2024-11-19T04:40:50,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/ed6a5536454647d2868d5a69ff37a5f2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ed6a5536454647d2868d5a69ff37a5f2 2024-11-19T04:40:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991310857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ed6a5536454647d2868d5a69ff37a5f2, entries=150, sequenceid=283, filesize=12.0 K 2024-11-19T04:40:50,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9825540f42ae426c97f37c90dec5f424 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9825540f42ae426c97f37c90dec5f424 2024-11-19T04:40:50,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991310867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:50,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9825540f42ae426c97f37c90dec5f424, entries=150, sequenceid=283, filesize=12.0 K 2024-11-19T04:40:50,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e97938390deb1164b875a27699276aad in 1286ms, sequenceid=283, compaction requested=true 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:50,872 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:50,872 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:50,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:50,873 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:50,873 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:50,873 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:50,873 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:50,873 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,873 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,874 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/05ddd007bc3e4976aaed8a1aedfc4499, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/fd74a6f720a041e9890997294c59f1e8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/a8d3b13abd4b4f1da2f454449c363243] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.4 K 2024-11-19T04:40:50,874 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fdb9f2e30e8d4f87a0e5054632001eb9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e6ef3b16c4d54805aa7396d93ef706d8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ed6a5536454647d2868d5a69ff37a5f2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.4 K 2024-11-19T04:40:50,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb9f2e30e8d4f87a0e5054632001eb9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731991247034 2024-11-19T04:40:50,874 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ddd007bc3e4976aaed8a1aedfc4499, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731991247034 2024-11-19T04:40:50,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ef3b16c4d54805aa7396d93ef706d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731991247419 2024-11-19T04:40:50,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd74a6f720a041e9890997294c59f1e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731991247419 2024-11-19T04:40:50,875 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6a5536454647d2868d5a69ff37a5f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731991249580 2024-11-19T04:40:50,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d3b13abd4b4f1da2f454449c363243, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731991249580 2024-11-19T04:40:50,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T04:40:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,878 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T04:40:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:50,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/cd49157e7aa949d19e06bbe329ca9b1e is 50, key is test_row_0/A:col10/1731991249717/Put/seqid=0 2024-11-19T04:40:50,886 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:50,887 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/a698561ad8f6461591457bc6301c96fb is 50, key is test_row_0/B:col10/1731991249584/Put/seqid=0 2024-11-19T04:40:50,890 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:50,891 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/14d327dd330548b3887682c3b454a020 is 50, key is test_row_0/A:col10/1731991249584/Put/seqid=0 2024-11-19T04:40:50,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741894_1070 (size=12301) 2024-11-19T04:40:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741895_1071 (size=12949) 2024-11-19T04:40:50,911 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/a698561ad8f6461591457bc6301c96fb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a698561ad8f6461591457bc6301c96fb 2024-11-19T04:40:50,920 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into a698561ad8f6461591457bc6301c96fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:50,920 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:50,920 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=13, startTime=1731991250872; duration=0sec 2024-11-19T04:40:50,920 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:50,921 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:50,921 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:50,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741896_1072 (size=12949) 2024-11-19T04:40:50,924 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:50,924 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:50,924 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:50,924 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8e7479ff306b4ae4b70f8ecb2f7a2439, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f6fb404d5d73438fae4346d11c2c288f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9825540f42ae426c97f37c90dec5f424] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.4 K 2024-11-19T04:40:50,925 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e7479ff306b4ae4b70f8ecb2f7a2439, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731991247034 2024-11-19T04:40:50,925 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f6fb404d5d73438fae4346d11c2c288f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731991247419 2024-11-19T04:40:50,926 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9825540f42ae426c97f37c90dec5f424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731991249580 2024-11-19T04:40:50,930 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/14d327dd330548b3887682c3b454a020 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/14d327dd330548b3887682c3b454a020 2024-11-19T04:40:50,940 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into 14d327dd330548b3887682c3b454a020(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:50,940 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:50,940 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=13, startTime=1731991250872; duration=0sec 2024-11-19T04:40:50,940 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:50,940 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:50,942 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:50,943 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/05a38fbd5d6848dc92b686bc6e9ca49a is 50, key is test_row_0/C:col10/1731991249584/Put/seqid=0 2024-11-19T04:40:50,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741897_1073 (size=12949) 2024-11-19T04:40:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T04:40:51,304 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/cd49157e7aa949d19e06bbe329ca9b1e 2024-11-19T04:40:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b66229db26594282a459d4c23c80cbc0 is 50, key is test_row_0/B:col10/1731991249717/Put/seqid=0 2024-11-19T04:40:51,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741898_1074 (size=12301) 2024-11-19T04:40:51,321 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b66229db26594282a459d4c23c80cbc0 2024-11-19T04:40:51,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/8f0c03b94a534f2d958f602784b8fcbe is 50, key is test_row_0/C:col10/1731991249717/Put/seqid=0 2024-11-19T04:40:51,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741899_1075 (size=12301) 2024-11-19T04:40:51,363 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/05a38fbd5d6848dc92b686bc6e9ca49a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/05a38fbd5d6848dc92b686bc6e9ca49a 2024-11-19T04:40:51,371 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into 05a38fbd5d6848dc92b686bc6e9ca49a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:51,371 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:51,371 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=13, startTime=1731991250872; duration=0sec 2024-11-19T04:40:51,372 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:51,372 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:51,743 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/8f0c03b94a534f2d958f602784b8fcbe 2024-11-19T04:40:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/cd49157e7aa949d19e06bbe329ca9b1e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/cd49157e7aa949d19e06bbe329ca9b1e 2024-11-19T04:40:51,761 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/cd49157e7aa949d19e06bbe329ca9b1e, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T04:40:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/b66229db26594282a459d4c23c80cbc0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b66229db26594282a459d4c23c80cbc0 2024-11-19T04:40:51,769 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b66229db26594282a459d4c23c80cbc0, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T04:40:51,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/8f0c03b94a534f2d958f602784b8fcbe as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8f0c03b94a534f2d958f602784b8fcbe 2024-11-19T04:40:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8f0c03b94a534f2d958f602784b8fcbe, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T04:40:51,779 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for e97938390deb1164b875a27699276aad in 900ms, sequenceid=319, compaction requested=false 2024-11-19T04:40:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T04:40:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T04:40:51,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T04:40:51,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-11-19T04:40:51,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.8470 sec 2024-11-19T04:40:51,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:40:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:51,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/54ba1be92c674738b0e4ca1a9a4a7c38 is 50, key is test_row_0/A:col10/1731991251842/Put/seqid=0 2024-11-19T04:40:51,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741900_1076 (size=7415) 2024-11-19T04:40:51,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991251880, max timestamp=1731991251895Num uniques 859; 2024-11-19T04:40:51,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:40:51,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991251880, max timestamp=1731991251895Num uniques 859; 2024-11-19T04:40:51,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:40:51,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731991251881, max timestamp=1731991251895Num uniques 859; 2024-11-19T04:40:51,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T04:40:51,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991311924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991311925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991311927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991311938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991311938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991312048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991312049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991312049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991312050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991312050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T04:40:52,088 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:52,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T04:40:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T04:40:52,091 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:52,092 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:52,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T04:40:52,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T04:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991312257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991312257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991312257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991312257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991312258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/54ba1be92c674738b0e4ca1a9a4a7c38 2024-11-19T04:40:52,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/1cea150a91b24f14833d096da90795ea is 50, key is test_row_0/B:col10/1731991251842/Put/seqid=0 2024-11-19T04:40:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741901_1077 (size=7415) 2024-11-19T04:40:52,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/1cea150a91b24f14833d096da90795ea 2024-11-19T04:40:52,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/1b200d6544564e48bd8e89c8615126e2 is 50, key is test_row_0/C:col10/1731991251842/Put/seqid=0 2024-11-19T04:40:52,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741902_1078 (size=7415) 2024-11-19T04:40:52,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T04:40:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T04:40:52,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T04:40:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991312567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991312567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991312577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991312577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991312577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:52,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T04:40:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T04:40:52,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/1b200d6544564e48bd8e89c8615126e2 2024-11-19T04:40:52,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/54ba1be92c674738b0e4ca1a9a4a7c38 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/54ba1be92c674738b0e4ca1a9a4a7c38 2024-11-19T04:40:52,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/54ba1be92c674738b0e4ca1a9a4a7c38, entries=50, sequenceid=332, filesize=7.2 K 2024-11-19T04:40:52,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/1cea150a91b24f14833d096da90795ea as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1cea150a91b24f14833d096da90795ea 2024-11-19T04:40:52,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1cea150a91b24f14833d096da90795ea, entries=50, sequenceid=332, filesize=7.2 K 2024-11-19T04:40:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/1b200d6544564e48bd8e89c8615126e2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/1b200d6544564e48bd8e89c8615126e2 2024-11-19T04:40:52,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/1b200d6544564e48bd8e89c8615126e2, entries=50, sequenceid=332, filesize=7.2 K 2024-11-19T04:40:52,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.26 KB/141575 for e97938390deb1164b875a27699276aad in 938ms, sequenceid=332, compaction requested=true 2024-11-19T04:40:52,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:52,783 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:52,783 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:52,784 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:52,784 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:52,784 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,785 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/14d327dd330548b3887682c3b454a020, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/cd49157e7aa949d19e06bbe329ca9b1e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/54ba1be92c674738b0e4ca1a9a4a7c38] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=31.9 K 2024-11-19T04:40:52,786 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:52,786 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d327dd330548b3887682c3b454a020, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731991249580 2024-11-19T04:40:52,786 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:52,786 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,786 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a698561ad8f6461591457bc6301c96fb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b66229db26594282a459d4c23c80cbc0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1cea150a91b24f14833d096da90795ea] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=31.9 K 2024-11-19T04:40:52,786 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd49157e7aa949d19e06bbe329ca9b1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731991249700 2024-11-19T04:40:52,787 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a698561ad8f6461591457bc6301c96fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731991249580 2024-11-19T04:40:52,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54ba1be92c674738b0e4ca1a9a4a7c38, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731991251842 2024-11-19T04:40:52,788 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b66229db26594282a459d4c23c80cbc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731991249700 2024-11-19T04:40:52,789 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cea150a91b24f14833d096da90795ea, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731991251842 2024-11-19T04:40:52,810 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:52,811 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e85b34feb1d447e098c9c4a22683b231 is 50, key is test_row_0/A:col10/1731991251842/Put/seqid=0 2024-11-19T04:40:52,818 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:52,823 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/65d48d458578450992647dae1bc03a62 is 50, key is test_row_0/B:col10/1731991251842/Put/seqid=0 2024-11-19T04:40:52,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T04:40:52,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:52,863 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=138.26 KB heapSize=394.23 KB 2024-11-19T04:40:52,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:52,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:52,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:52,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:52,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:52,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:52,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741904_1080 (size=13051) 2024-11-19T04:40:52,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741903_1079 (size=13051) 2024-11-19T04:40:52,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/2bc4b51bd377411ea0f4561b127f71dd is 50, key is test_row_0/A:col10/1731991251935/Put/seqid=0 2024-11-19T04:40:52,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741905_1081 (size=12301) 2024-11-19T04:40:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:53,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:53,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991313092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991313097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991313097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991313098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991313100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991313208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991313217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T04:40:53,281 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/2bc4b51bd377411ea0f4561b127f71dd 2024-11-19T04:40:53,296 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e85b34feb1d447e098c9c4a22683b231 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e85b34feb1d447e098c9c4a22683b231 2024-11-19T04:40:53,297 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/65d48d458578450992647dae1bc03a62 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/65d48d458578450992647dae1bc03a62 2024-11-19T04:40:53,308 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into 65d48d458578450992647dae1bc03a62(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:53,308 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:53,308 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=13, startTime=1731991252783; duration=0sec 2024-11-19T04:40:53,308 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:53,308 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:53,308 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:53,309 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into e85b34feb1d447e098c9c4a22683b231(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:53,309 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:53,309 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=13, startTime=1731991252782; duration=0sec 2024-11-19T04:40:53,309 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:53,309 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:53,312 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:53,312 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:53,313 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:53,313 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/05a38fbd5d6848dc92b686bc6e9ca49a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8f0c03b94a534f2d958f602784b8fcbe, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/1b200d6544564e48bd8e89c8615126e2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=31.9 K 2024-11-19T04:40:53,313 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a38fbd5d6848dc92b686bc6e9ca49a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731991249580 2024-11-19T04:40:53,316 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0c03b94a534f2d958f602784b8fcbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731991249700 2024-11-19T04:40:53,317 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b200d6544564e48bd8e89c8615126e2, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731991251842 2024-11-19T04:40:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e68fb8e53f484f87bd7d1220be2573ce is 50, key is test_row_0/B:col10/1731991251935/Put/seqid=0 2024-11-19T04:40:53,343 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:53,344 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/46f635d7c34c48bdb9dd92b82d6e7671 is 50, key is test_row_0/C:col10/1731991251842/Put/seqid=0 2024-11-19T04:40:53,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741906_1082 (size=12301) 2024-11-19T04:40:53,364 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e68fb8e53f484f87bd7d1220be2573ce 2024-11-19T04:40:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f9ad5e76c11d4a69a38facf6f14eeedc is 50, key is test_row_0/C:col10/1731991251935/Put/seqid=0 2024-11-19T04:40:53,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741907_1083 (size=13051) 2024-11-19T04:40:53,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741908_1084 (size=12301) 2024-11-19T04:40:53,397 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f9ad5e76c11d4a69a38facf6f14eeedc 2024-11-19T04:40:53,398 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/46f635d7c34c48bdb9dd92b82d6e7671 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/46f635d7c34c48bdb9dd92b82d6e7671 2024-11-19T04:40:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/2bc4b51bd377411ea0f4561b127f71dd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2bc4b51bd377411ea0f4561b127f71dd 2024-11-19T04:40:53,417 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2bc4b51bd377411ea0f4561b127f71dd, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T04:40:53,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/e68fb8e53f484f87bd7d1220be2573ce as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e68fb8e53f484f87bd7d1220be2573ce 2024-11-19T04:40:53,419 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into 46f635d7c34c48bdb9dd92b82d6e7671(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:53,420 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:53,420 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=13, startTime=1731991252783; duration=0sec 2024-11-19T04:40:53,420 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:53,420 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991313417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,427 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e68fb8e53f484f87bd7d1220be2573ce, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T04:40:53,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/f9ad5e76c11d4a69a38facf6f14eeedc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f9ad5e76c11d4a69a38facf6f14eeedc 2024-11-19T04:40:53,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991313428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,435 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f9ad5e76c11d4a69a38facf6f14eeedc, entries=150, sequenceid=369, filesize=12.0 K 2024-11-19T04:40:53,437 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~138.26 KB/141575, heapSize ~394.18 KB/403640, currentSize=40.25 KB/41220 for e97938390deb1164b875a27699276aad in 574ms, sequenceid=369, compaction requested=false 2024-11-19T04:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T04:40:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T04:40:53,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T04:40:53,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2024-11-19T04:40:53,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3550 sec 2024-11-19T04:40:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:53,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:40:53,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:53,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:53,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:53,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:53,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:53,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:53,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/d28eedaa40a24e5183b25ea65a4c115a is 50, key is test_row_0/A:col10/1731991253728/Put/seqid=0 2024-11-19T04:40:53,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741909_1085 (size=14741) 2024-11-19T04:40:53,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/d28eedaa40a24e5183b25ea65a4c115a 2024-11-19T04:40:53,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/7396ceb465104df09898c5056440fb4d is 50, key is test_row_0/B:col10/1731991253728/Put/seqid=0 2024-11-19T04:40:53,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991253791, max timestamp=1731991253812Num uniques 859; 2024-11-19T04:40:53,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991253791, max timestamp=1731991253812Num uniques 859; 2024-11-19T04:40:53,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991253791, max timestamp=1731991253812Num uniques 859; 2024-11-19T04:40:53,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:40:53,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:40:53,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:40:53,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741910_1086 (size=12301) 2024-11-19T04:40:53,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/7396ceb465104df09898c5056440fb4d 2024-11-19T04:40:53,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/16a06c597dbc444fb3fe3fcbcd28c8be is 50, key is test_row_0/C:col10/1731991253728/Put/seqid=0 2024-11-19T04:40:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741911_1087 (size=12301) 2024-11-19T04:40:53,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/16a06c597dbc444fb3fe3fcbcd28c8be 2024-11-19T04:40:53,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/d28eedaa40a24e5183b25ea65a4c115a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/d28eedaa40a24e5183b25ea65a4c115a 2024-11-19T04:40:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991313920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/d28eedaa40a24e5183b25ea65a4c115a, entries=200, sequenceid=382, filesize=14.4 K 2024-11-19T04:40:53,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/7396ceb465104df09898c5056440fb4d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/7396ceb465104df09898c5056440fb4d 2024-11-19T04:40:53,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991313928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:53,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/7396ceb465104df09898c5056440fb4d, entries=150, sequenceid=382, filesize=12.0 K 2024-11-19T04:40:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/16a06c597dbc444fb3fe3fcbcd28c8be as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/16a06c597dbc444fb3fe3fcbcd28c8be 2024-11-19T04:40:53,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/16a06c597dbc444fb3fe3fcbcd28c8be, entries=150, sequenceid=382, filesize=12.0 K 2024-11-19T04:40:53,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e97938390deb1164b875a27699276aad in 217ms, sequenceid=382, compaction requested=true 2024-11-19T04:40:53,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:53,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:53,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:53,949 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:53,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:53,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:53,950 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:53,951 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:53,951 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:53,951 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:53,951 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:53,951 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:53,951 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:53,951 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/65d48d458578450992647dae1bc03a62, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e68fb8e53f484f87bd7d1220be2573ce, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/7396ceb465104df09898c5056440fb4d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.8 K 2024-11-19T04:40:53,951 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e85b34feb1d447e098c9c4a22683b231, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2bc4b51bd377411ea0f4561b127f71dd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/d28eedaa40a24e5183b25ea65a4c115a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=39.2 K 2024-11-19T04:40:53,952 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d48d458578450992647dae1bc03a62, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731991249700 2024-11-19T04:40:53,952 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e85b34feb1d447e098c9c4a22683b231, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731991249700 2024-11-19T04:40:53,953 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e68fb8e53f484f87bd7d1220be2573ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731991251919 2024-11-19T04:40:53,953 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc4b51bd377411ea0f4561b127f71dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731991251919 2024-11-19T04:40:53,953 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 7396ceb465104df09898c5056440fb4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731991253090 2024-11-19T04:40:53,954 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28eedaa40a24e5183b25ea65a4c115a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731991253083 2024-11-19T04:40:53,988 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:53,989 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/68256b52df7f48a49bac429f2e1fdd00 is 50, key is test_row_0/B:col10/1731991253728/Put/seqid=0 2024-11-19T04:40:53,999 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:53,999 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/ba1a366c7e454d338443405e6e394991 is 50, key is test_row_0/A:col10/1731991253728/Put/seqid=0 2024-11-19T04:40:54,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741913_1089 (size=13153) 2024-11-19T04:40:54,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741912_1088 (size=13153) 2024-11-19T04:40:54,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731991253898, max timestamp=1731991254042Num uniques 1018; 2024-11-19T04:40:54,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731991253898, max timestamp=1731991254042Num uniques 1018; 2024-11-19T04:40:54,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T04:40:54,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-19T04:40:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:54,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-19T04:40:54,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:40:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:54,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T04:40:54,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/bb474f6c7bdb49318e1d063dad3135e5 is 50, key is test_row_0/A:col10/1731991253915/Put/seqid=0 2024-11-19T04:40:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741914_1090 (size=14741) 2024-11-19T04:40:54,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/bb474f6c7bdb49318e1d063dad3135e5 2024-11-19T04:40:54,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/23c9b4ea187545abb417a7144609615b is 50, key is test_row_0/B:col10/1731991253915/Put/seqid=0 2024-11-19T04:40:54,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741915_1091 (size=12301) 2024-11-19T04:40:54,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991314120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991314121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991314123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991314128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991314130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T04:40:54,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:54,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-19T04:40:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T04:40:54,241 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:54,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:54,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:54,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991314237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991314237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991314237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991314247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991314248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T04:40:54,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T04:40:54,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:54,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,422 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/ba1a366c7e454d338443405e6e394991 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ba1a366c7e454d338443405e6e394991 2024-11-19T04:40:54,428 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/68256b52df7f48a49bac429f2e1fdd00 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/68256b52df7f48a49bac429f2e1fdd00 2024-11-19T04:40:54,434 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into ba1a366c7e454d338443405e6e394991(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:54,434 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:54,434 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=13, startTime=1731991253949; duration=0sec 2024-11-19T04:40:54,434 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:54,435 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:54,435 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:54,440 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:54,441 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:54,441 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,441 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/46f635d7c34c48bdb9dd92b82d6e7671, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f9ad5e76c11d4a69a38facf6f14eeedc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/16a06c597dbc444fb3fe3fcbcd28c8be] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=36.8 K 2024-11-19T04:40:54,442 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into 68256b52df7f48a49bac429f2e1fdd00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:54,442 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:54,442 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=13, startTime=1731991253949; duration=0sec 2024-11-19T04:40:54,442 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:54,442 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:54,443 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f635d7c34c48bdb9dd92b82d6e7671, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731991249700 2024-11-19T04:40:54,444 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ad5e76c11d4a69a38facf6f14eeedc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731991251919 2024-11-19T04:40:54,444 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16a06c597dbc444fb3fe3fcbcd28c8be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731991253090 2024-11-19T04:40:54,468 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:54,469 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/6906a0534ad74649ba3be36516dfbffb is 50, key is test_row_0/C:col10/1731991253728/Put/seqid=0 2024-11-19T04:40:54,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991314457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991314457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991314457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991314459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991314467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741916_1092 (size=13153) 2024-11-19T04:40:54,511 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/6906a0534ad74649ba3be36516dfbffb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/6906a0534ad74649ba3be36516dfbffb 2024-11-19T04:40:54,521 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into 6906a0534ad74649ba3be36516dfbffb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:54,521 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:54,521 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=13, startTime=1731991253950; duration=0sec 2024-11-19T04:40:54,521 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:54,521 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:54,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/23c9b4ea187545abb417a7144609615b 2024-11-19T04:40:54,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T04:40:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:54,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T04:40:54,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9cb9962b80ab4a9ba96aede9ef06f89a is 50, key is test_row_0/C:col10/1731991253915/Put/seqid=0 2024-11-19T04:40:54,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741917_1093 (size=12301) 2024-11-19T04:40:54,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T04:40:54,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:54,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991314777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991314777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991314777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991314779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991314788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:54,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T04:40:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T04:40:54,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:55,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9cb9962b80ab4a9ba96aede9ef06f89a 2024-11-19T04:40:55,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T04:40:55,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:55,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:55,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:55,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:55,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/bb474f6c7bdb49318e1d063dad3135e5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/bb474f6c7bdb49318e1d063dad3135e5 2024-11-19T04:40:55,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/bb474f6c7bdb49318e1d063dad3135e5, entries=200, sequenceid=420, filesize=14.4 K 2024-11-19T04:40:55,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/23c9b4ea187545abb417a7144609615b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/23c9b4ea187545abb417a7144609615b 2024-11-19T04:40:55,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/23c9b4ea187545abb417a7144609615b, entries=150, sequenceid=420, filesize=12.0 K 2024-11-19T04:40:55,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9cb9962b80ab4a9ba96aede9ef06f89a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9cb9962b80ab4a9ba96aede9ef06f89a 2024-11-19T04:40:55,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9cb9962b80ab4a9ba96aede9ef06f89a, entries=150, sequenceid=420, filesize=12.0 K 2024-11-19T04:40:55,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=114.05 KB/116790 for e97938390deb1164b875a27699276aad in 1016ms, sequenceid=420, compaction requested=false 2024-11-19T04:40:55,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:55,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-19T04:40:55,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:55,179 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:55,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/549acfee402e4188888c63ebf63fa36c is 50, key is test_row_0/A:col10/1731991254093/Put/seqid=0 2024-11-19T04:40:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741918_1094 (size=12301) 2024-11-19T04:40:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:55,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:55,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991315302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991315306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991315308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991315309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991315317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T04:40:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991315417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991315417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991315427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991315427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,595 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/549acfee402e4188888c63ebf63fa36c 2024-11-19T04:40:55,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/f98b420fb41f4404a5bb240265c79efc is 50, key is test_row_0/B:col10/1731991254093/Put/seqid=0 2024-11-19T04:40:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991315627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991315628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991315637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991315639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741919_1095 (size=12301) 2024-11-19T04:40:55,643 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/f98b420fb41f4404a5bb240265c79efc 2024-11-19T04:40:55,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9f649c15af734774b79153de59a148bb is 50, key is test_row_0/C:col10/1731991254093/Put/seqid=0 2024-11-19T04:40:55,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741920_1096 (size=12301) 2024-11-19T04:40:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991315937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991315937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991315947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991315947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:55,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,081 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9f649c15af734774b79153de59a148bb 2024-11-19T04:40:56,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/549acfee402e4188888c63ebf63fa36c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/549acfee402e4188888c63ebf63fa36c 2024-11-19T04:40:56,094 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/549acfee402e4188888c63ebf63fa36c, entries=150, sequenceid=443, filesize=12.0 K 2024-11-19T04:40:56,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/f98b420fb41f4404a5bb240265c79efc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/f98b420fb41f4404a5bb240265c79efc 2024-11-19T04:40:56,101 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/f98b420fb41f4404a5bb240265c79efc, entries=150, sequenceid=443, filesize=12.0 K 2024-11-19T04:40:56,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/9f649c15af734774b79153de59a148bb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9f649c15af734774b79153de59a148bb 2024-11-19T04:40:56,119 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9f649c15af734774b79153de59a148bb, entries=150, sequenceid=443, filesize=12.0 K 2024-11-19T04:40:56,120 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e97938390deb1164b875a27699276aad in 941ms, sequenceid=443, compaction requested=true 2024-11-19T04:40:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-19T04:40:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-19T04:40:56,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T04:40:56,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-19T04:40:56,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8870 sec 2024-11-19T04:40:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:56,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:40:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:56,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:56,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/87570a0c5fe94ea58de92bbdefa795a4 is 50, key is test_row_0/A:col10/1731991256338/Put/seqid=0 2024-11-19T04:40:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T04:40:56,387 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:56,388 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-19T04:40:56,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:56,392 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:56,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:56,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741921_1097 (size=12301) 2024-11-19T04:40:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T04:40:56,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991316443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991316456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991316457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991316457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991316457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T04:40:56,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T04:40:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991316559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T04:40:56,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T04:40:56,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991316767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:56,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/87570a0c5fe94ea58de92bbdefa795a4 2024-11-19T04:40:56,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/5c56317c616d4c72ac8a156282923b58 is 50, key is test_row_0/B:col10/1731991256338/Put/seqid=0 2024-11-19T04:40:56,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741922_1098 (size=12301) 2024-11-19T04:40:56,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/5c56317c616d4c72ac8a156282923b58 2024-11-19T04:40:56,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/252add34272a4346a0b3eacac808b657 is 50, key is test_row_0/C:col10/1731991256338/Put/seqid=0 2024-11-19T04:40:56,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741923_1099 (size=12301) 2024-11-19T04:40:56,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T04:40:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:56,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T04:40:57,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T04:40:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991317077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T04:40:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/252add34272a4346a0b3eacac808b657 2024-11-19T04:40:57,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/87570a0c5fe94ea58de92bbdefa795a4 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/87570a0c5fe94ea58de92bbdefa795a4 2024-11-19T04:40:57,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/87570a0c5fe94ea58de92bbdefa795a4, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T04:40:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/5c56317c616d4c72ac8a156282923b58 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/5c56317c616d4c72ac8a156282923b58 2024-11-19T04:40:57,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/5c56317c616d4c72ac8a156282923b58, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T04:40:57,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/252add34272a4346a0b3eacac808b657 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/252add34272a4346a0b3eacac808b657 2024-11-19T04:40:57,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/252add34272a4346a0b3eacac808b657, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T04:40:57,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e97938390deb1164b875a27699276aad in 955ms, sequenceid=456, compaction requested=true 2024-11-19T04:40:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:57,296 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:57,296 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:57,300 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:57,300 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:57,300 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,300 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/68256b52df7f48a49bac429f2e1fdd00, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/23c9b4ea187545abb417a7144609615b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/f98b420fb41f4404a5bb240265c79efc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/5c56317c616d4c72ac8a156282923b58] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=48.9 K 2024-11-19T04:40:57,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:57,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:57,301 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,301 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 68256b52df7f48a49bac429f2e1fdd00, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731991253090 2024-11-19T04:40:57,301 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ba1a366c7e454d338443405e6e394991, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/bb474f6c7bdb49318e1d063dad3135e5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/549acfee402e4188888c63ebf63fa36c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/87570a0c5fe94ea58de92bbdefa795a4] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=51.3 K 2024-11-19T04:40:57,302 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c9b4ea187545abb417a7144609615b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731991253915 2024-11-19T04:40:57,302 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1a366c7e454d338443405e6e394991, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731991253090 2024-11-19T04:40:57,302 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f98b420fb41f4404a5bb240265c79efc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731991254093 2024-11-19T04:40:57,302 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb474f6c7bdb49318e1d063dad3135e5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731991253898 2024-11-19T04:40:57,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c56317c616d4c72ac8a156282923b58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991255306 2024-11-19T04:40:57,306 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549acfee402e4188888c63ebf63fa36c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731991254093 2024-11-19T04:40:57,308 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87570a0c5fe94ea58de92bbdefa795a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991255306 2024-11-19T04:40:57,320 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:57,321 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/0a1b58c2f671431d993f26c8305b1443 is 50, key is test_row_0/B:col10/1731991256338/Put/seqid=0 2024-11-19T04:40:57,325 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:57,326 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/26fb3e6090304eaa9348855fcbb94090 is 50, key is test_row_0/A:col10/1731991256338/Put/seqid=0 2024-11-19T04:40:57,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-19T04:40:57,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,330 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T04:40:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741924_1100 (size=13289) 2024-11-19T04:40:57,348 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/0a1b58c2f671431d993f26c8305b1443 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/0a1b58c2f671431d993f26c8305b1443 2024-11-19T04:40:57,356 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into 0a1b58c2f671431d993f26c8305b1443(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:57,356 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:57,356 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=12, startTime=1731991257296; duration=0sec 2024-11-19T04:40:57,357 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:57,357 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:57,357 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:40:57,359 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:40:57,359 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:57,359 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,359 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/6906a0534ad74649ba3be36516dfbffb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9cb9962b80ab4a9ba96aede9ef06f89a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9f649c15af734774b79153de59a148bb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/252add34272a4346a0b3eacac808b657] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=48.9 K 2024-11-19T04:40:57,360 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6906a0534ad74649ba3be36516dfbffb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731991253090 2024-11-19T04:40:57,360 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb9962b80ab4a9ba96aede9ef06f89a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731991253915 2024-11-19T04:40:57,361 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f649c15af734774b79153de59a148bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731991254093 2024-11-19T04:40:57,361 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 252add34272a4346a0b3eacac808b657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991255306 2024-11-19T04:40:57,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741925_1101 (size=13289) 2024-11-19T04:40:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/5c6016378dd649779eaa283ebb14b89e is 50, key is test_row_0/A:col10/1731991256414/Put/seqid=0 2024-11-19T04:40:57,383 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/26fb3e6090304eaa9348855fcbb94090 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/26fb3e6090304eaa9348855fcbb94090 2024-11-19T04:40:57,394 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into 26fb3e6090304eaa9348855fcbb94090(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:57,394 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:57,394 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=12, startTime=1731991257296; duration=0sec 2024-11-19T04:40:57,394 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:57,395 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:57,397 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#90 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:57,398 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/a6b2a4765bef4dd1bc551f34a51f7c05 is 50, key is test_row_0/C:col10/1731991256338/Put/seqid=0 2024-11-19T04:40:57,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741926_1102 (size=12301) 2024-11-19T04:40:57,404 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/5c6016378dd649779eaa283ebb14b89e 2024-11-19T04:40:57,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741927_1103 (size=13289) 2024-11-19T04:40:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/eccdf45d729441cf8cbf4a2540589524 is 50, key is test_row_0/B:col10/1731991256414/Put/seqid=0 2024-11-19T04:40:57,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741928_1104 (size=12301) 2024-11-19T04:40:57,447 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/eccdf45d729441cf8cbf4a2540589524 2024-11-19T04:40:57,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/3a2d1fa66c6c4d55ad2c5a05d5131af3 is 50, key is test_row_0/C:col10/1731991256414/Put/seqid=0 2024-11-19T04:40:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:57,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741929_1105 (size=12301) 2024-11-19T04:40:57,476 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/3a2d1fa66c6c4d55ad2c5a05d5131af3 2024-11-19T04:40:57,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/5c6016378dd649779eaa283ebb14b89e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/5c6016378dd649779eaa283ebb14b89e 2024-11-19T04:40:57,489 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/5c6016378dd649779eaa283ebb14b89e, entries=150, sequenceid=475, filesize=12.0 K 2024-11-19T04:40:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/eccdf45d729441cf8cbf4a2540589524 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/eccdf45d729441cf8cbf4a2540589524 2024-11-19T04:40:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991317493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,499 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/eccdf45d729441cf8cbf4a2540589524, entries=150, sequenceid=475, filesize=12.0 K 2024-11-19T04:40:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/3a2d1fa66c6c4d55ad2c5a05d5131af3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3a2d1fa66c6c4d55ad2c5a05d5131af3 2024-11-19T04:40:57,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991317495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991317496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991317497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,512 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3a2d1fa66c6c4d55ad2c5a05d5131af3, entries=150, sequenceid=475, filesize=12.0 K 2024-11-19T04:40:57,513 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e97938390deb1164b875a27699276aad in 183ms, sequenceid=475, compaction requested=false 2024-11-19T04:40:57,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:57,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-19T04:40:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-19T04:40:57,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T04:40:57,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1230 sec 2024-11-19T04:40:57,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.1330 sec 2024-11-19T04:40:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T04:40:57,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:57,538 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:40:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-19T04:40:57,541 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:40:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T04:40:57,551 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:40:57,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:40:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:57,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-19T04:40:57,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:57,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:57,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:57,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:57,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:57,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:57,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/4cbd6105777a4eea8574ecf2ac75f280 is 50, key is test_row_0/A:col10/1731991257490/Put/seqid=0 2024-11-19T04:40:57,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741930_1106 (size=14741) 2024-11-19T04:40:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T04:40:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991317672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991317674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991317675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991317675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991317677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T04:40:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991317787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991317787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991317787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991317797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991317798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:57,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:57,846 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/a6b2a4765bef4dd1bc551f34a51f7c05 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a6b2a4765bef4dd1bc551f34a51f7c05 2024-11-19T04:40:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T04:40:57,857 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into a6b2a4765bef4dd1bc551f34a51f7c05(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:57,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:57,857 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=12, startTime=1731991257296; duration=0sec 2024-11-19T04:40:57,858 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:57,858 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:57,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T04:40:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:57,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991317998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991317998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991317998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T04:40:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:40:58,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/4cbd6105777a4eea8574ecf2ac75f280 2024-11-19T04:40:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991318037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991318037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/03b9b376f4744b5fb959f433135e0517 is 50, key is test_row_0/B:col10/1731991257490/Put/seqid=0 2024-11-19T04:40:58,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741931_1107 (size=12301) 2024-11-19T04:40:58,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/03b9b376f4744b5fb959f433135e0517 2024-11-19T04:40:58,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/a69dfd165f25487889ce01cc5356cc8b is 50, key is test_row_0/C:col10/1731991257490/Put/seqid=0 2024-11-19T04:40:58,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741932_1108 (size=12301) 2024-11-19T04:40:58,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=492 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/a69dfd165f25487889ce01cc5356cc8b 2024-11-19T04:40:58,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/4cbd6105777a4eea8574ecf2ac75f280 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/4cbd6105777a4eea8574ecf2ac75f280 2024-11-19T04:40:58,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/4cbd6105777a4eea8574ecf2ac75f280, entries=200, sequenceid=492, filesize=14.4 K 2024-11-19T04:40:58,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/03b9b376f4744b5fb959f433135e0517 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/03b9b376f4744b5fb959f433135e0517 2024-11-19T04:40:58,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/03b9b376f4744b5fb959f433135e0517, entries=150, sequenceid=492, filesize=12.0 K 2024-11-19T04:40:58,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/a69dfd165f25487889ce01cc5356cc8b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a69dfd165f25487889ce01cc5356cc8b 2024-11-19T04:40:58,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a69dfd165f25487889ce01cc5356cc8b, entries=150, sequenceid=492, filesize=12.0 K 2024-11-19T04:40:58,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e97938390deb1164b875a27699276aad in 541ms, sequenceid=492, compaction requested=true 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:58,143 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:58,143 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:40:58,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:58,144 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:58,144 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:58,144 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:40:58,144 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:40:58,144 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,144 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,144 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/0a1b58c2f671431d993f26c8305b1443, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/eccdf45d729441cf8cbf4a2540589524, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/03b9b376f4744b5fb959f433135e0517] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=37.0 K 2024-11-19T04:40:58,145 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/26fb3e6090304eaa9348855fcbb94090, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/5c6016378dd649779eaa283ebb14b89e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/4cbd6105777a4eea8574ecf2ac75f280] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=39.4 K 2024-11-19T04:40:58,145 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a1b58c2f671431d993f26c8305b1443, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991255306 2024-11-19T04:40:58,145 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26fb3e6090304eaa9348855fcbb94090, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991255306 2024-11-19T04:40:58,145 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting eccdf45d729441cf8cbf4a2540589524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731991256414 2024-11-19T04:40:58,145 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c6016378dd649779eaa283ebb14b89e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731991256414 2024-11-19T04:40:58,146 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b9b376f4744b5fb959f433135e0517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731991257490 2024-11-19T04:40:58,146 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbd6105777a4eea8574ecf2ac75f280, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731991257483 2024-11-19T04:40:58,162 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:58,162 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/66b5dea6478f4c5a920ecaf93c4ebb4f is 50, key is test_row_0/B:col10/1731991257490/Put/seqid=0 2024-11-19T04:40:58,163 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:58,164 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/dc88598c1f504ff2a3bebd65d24b8745 is 50, key is test_row_0/A:col10/1731991257490/Put/seqid=0 2024-11-19T04:40:58,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,167 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:58,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e1e00ff88a444588948bcb3e6ce09163 is 50, key is test_row_0/A:col10/1731991257674/Put/seqid=0 2024-11-19T04:40:58,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741934_1110 (size=13391) 2024-11-19T04:40:58,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741933_1109 (size=13391) 2024-11-19T04:40:58,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741935_1111 (size=12301) 2024-11-19T04:40:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:58,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. as already flushing 2024-11-19T04:40:58,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991318342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991318350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991318350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991318358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991318358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991318457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991318468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991318468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,596 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/dc88598c1f504ff2a3bebd65d24b8745 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dc88598c1f504ff2a3bebd65d24b8745 2024-11-19T04:40:58,609 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/66b5dea6478f4c5a920ecaf93c4ebb4f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/66b5dea6478f4c5a920ecaf93c4ebb4f 2024-11-19T04:40:58,616 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e1e00ff88a444588948bcb3e6ce09163 2024-11-19T04:40:58,619 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into 66b5dea6478f4c5a920ecaf93c4ebb4f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:58,619 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:58,619 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=13, startTime=1731991258143; duration=0sec 2024-11-19T04:40:58,619 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:40:58,619 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:40:58,619 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:40:58,622 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into dc88598c1f504ff2a3bebd65d24b8745(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:58,622 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:58,622 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=13, startTime=1731991258143; duration=0sec 2024-11-19T04:40:58,622 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:58,623 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:40:58,633 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:40:58,634 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:40:58,634 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,634 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a6b2a4765bef4dd1bc551f34a51f7c05, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3a2d1fa66c6c4d55ad2c5a05d5131af3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a69dfd165f25487889ce01cc5356cc8b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=37.0 K 2024-11-19T04:40:58,637 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a6b2a4765bef4dd1bc551f34a51f7c05, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991255306 2024-11-19T04:40:58,638 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a2d1fa66c6c4d55ad2c5a05d5131af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731991256414 2024-11-19T04:40:58,639 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a69dfd165f25487889ce01cc5356cc8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731991257490 2024-11-19T04:40:58,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/d5912087d6184d678db14e8252df845f is 50, key is test_row_0/B:col10/1731991257674/Put/seqid=0 2024-11-19T04:40:58,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991318667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T04:40:58,680 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#100 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:40:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991318677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741936_1112 (size=12301) 2024-11-19T04:40:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991318677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,688 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/d5912087d6184d678db14e8252df845f 2024-11-19T04:40:58,698 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/c94e0d115b794ce682661aa5931ebe75 is 50, key is test_row_0/C:col10/1731991257490/Put/seqid=0 2024-11-19T04:40:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/b2bea343fbeb481b874821b799828f01 is 50, key is test_row_0/C:col10/1731991257674/Put/seqid=0 2024-11-19T04:40:58,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741937_1113 (size=13391) 2024-11-19T04:40:58,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741938_1114 (size=12301) 2024-11-19T04:40:58,741 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/b2bea343fbeb481b874821b799828f01 2024-11-19T04:40:58,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/e1e00ff88a444588948bcb3e6ce09163 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e1e00ff88a444588948bcb3e6ce09163 2024-11-19T04:40:58,762 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e1e00ff88a444588948bcb3e6ce09163, entries=150, sequenceid=511, filesize=12.0 K 2024-11-19T04:40:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/d5912087d6184d678db14e8252df845f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/d5912087d6184d678db14e8252df845f 2024-11-19T04:40:58,770 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/d5912087d6184d678db14e8252df845f, entries=150, sequenceid=511, filesize=12.0 K 2024-11-19T04:40:58,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/b2bea343fbeb481b874821b799828f01 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b2bea343fbeb481b874821b799828f01 2024-11-19T04:40:58,786 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b2bea343fbeb481b874821b799828f01, entries=150, sequenceid=511, filesize=12.0 K 2024-11-19T04:40:58,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-19T04:40:58,787 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for e97938390deb1164b875a27699276aad in 620ms, sequenceid=511, compaction requested=false 2024-11-19T04:40:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:40:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-19T04:40:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-19T04:40:58,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T04:40:58,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2024-11-19T04:40:58,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.2540 sec 2024-11-19T04:40:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:40:58,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T04:40:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:40:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:40:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:40:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:40:58,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/114ba3b66189434492e455754e3b6660 is 50, key is test_row_0/A:col10/1731991258869/Put/seqid=0 2024-11-19T04:40:58,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741939_1115 (size=12301) 2024-11-19T04:40:58,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991318956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991318964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991318977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991318987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:58,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991318987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,047 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T04:40:59,047 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T04:40:59,047 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:40:59,047 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:40:59,051 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T04:40:59,051 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:40:59,051 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T04:40:59,051 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:40:59,053 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,053 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,053 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,053 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,054 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:40:59,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:40:59,054 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:40:59,055 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:40:59,055 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:40:59,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991319067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991319077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,143 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/c94e0d115b794ce682661aa5931ebe75 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c94e0d115b794ce682661aa5931ebe75 2024-11-19T04:40:59,151 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into c94e0d115b794ce682661aa5931ebe75(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:40:59,151 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:40:59,151 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=13, startTime=1731991258143; duration=0sec 2024-11-19T04:40:59,151 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:40:59,151 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:40:59,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991319277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991319287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/114ba3b66189434492e455754e3b6660 2024-11-19T04:40:59,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/35d63e558ae141468e95736681b53247 is 50, key is test_row_0/B:col10/1731991258869/Put/seqid=0 2024-11-19T04:40:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741940_1116 (size=12301) 2024-11-19T04:40:59,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33414 deadline: 1731991319487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33430 deadline: 1731991319496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33408 deadline: 1731991319496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991319587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:40:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991319596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:40:59,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:40:59,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:40:59,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T04:40:59,688 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:40:59,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/35d63e558ae141468e95736681b53247 2024-11-19T04:40:59,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/dcb503fda8af4a5ba74d649838984698 is 50, key is test_row_0/C:col10/1731991258869/Put/seqid=0 2024-11-19T04:40:59,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741941_1117 (size=12301) 2024-11-19T04:41:00,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33472 deadline: 1731991320097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33444 deadline: 1731991320106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:00,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/dcb503fda8af4a5ba74d649838984698 2024-11-19T04:41:00,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/114ba3b66189434492e455754e3b6660 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/114ba3b66189434492e455754e3b6660 2024-11-19T04:41:00,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/114ba3b66189434492e455754e3b6660, entries=150, sequenceid=529, filesize=12.0 K 2024-11-19T04:41:00,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/35d63e558ae141468e95736681b53247 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/35d63e558ae141468e95736681b53247 2024-11-19T04:41:00,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/35d63e558ae141468e95736681b53247, entries=150, sequenceid=529, filesize=12.0 K 2024-11-19T04:41:00,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/dcb503fda8af4a5ba74d649838984698 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dcb503fda8af4a5ba74d649838984698 2024-11-19T04:41:00,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dcb503fda8af4a5ba74d649838984698, entries=150, sequenceid=529, filesize=12.0 K 2024-11-19T04:41:00,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e97938390deb1164b875a27699276aad in 1289ms, sequenceid=529, compaction requested=true 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:00,161 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e97938390deb1164b875a27699276aad:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:00,161 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:00,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:00,162 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:00,162 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:00,163 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/A is initiating minor compaction (all files) 2024-11-19T04:41:00,163 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/B is initiating minor compaction (all files) 2024-11-19T04:41:00,163 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/A in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:00,163 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/B in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:00,163 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dc88598c1f504ff2a3bebd65d24b8745, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e1e00ff88a444588948bcb3e6ce09163, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/114ba3b66189434492e455754e3b6660] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=37.1 K 2024-11-19T04:41:00,163 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/66b5dea6478f4c5a920ecaf93c4ebb4f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/d5912087d6184d678db14e8252df845f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/35d63e558ae141468e95736681b53247] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=37.1 K 2024-11-19T04:41:00,163 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc88598c1f504ff2a3bebd65d24b8745, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731991257490 2024-11-19T04:41:00,163 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b5dea6478f4c5a920ecaf93c4ebb4f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731991257490 2024-11-19T04:41:00,164 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1e00ff88a444588948bcb3e6ce09163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731991257670 2024-11-19T04:41:00,164 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d5912087d6184d678db14e8252df845f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731991257670 2024-11-19T04:41:00,164 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 114ba3b66189434492e455754e3b6660, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1731991258326 2024-11-19T04:41:00,164 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d63e558ae141468e95736681b53247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1731991258326 2024-11-19T04:41:00,174 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:00,174 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/dae33271fb53418a9c05e3052c061216 is 50, key is test_row_0/A:col10/1731991258869/Put/seqid=0 2024-11-19T04:41:00,177 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:00,178 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/43131c2f478746d5bac4e6d64d6060d4 is 50, key is test_row_0/B:col10/1731991258869/Put/seqid=0 2024-11-19T04:41:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741942_1118 (size=13493) 2024-11-19T04:41:00,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741943_1119 (size=13493) 2024-11-19T04:41:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on e97938390deb1164b875a27699276aad 2024-11-19T04:41:00,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T04:41:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:41:00,499 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T04:41:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:41:00,499 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:41:00,499 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:00,499 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:00,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:00,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/1094c3bc63984f85937148807b20d5fb is 50, key is test_row_0/A:col10/1731991258940/Put/seqid=0 2024-11-19T04:41:00,510 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T04:41:00,510 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:00,510 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:00,510 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:00,510 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T04:41:00,510 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:00,510 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:00,510 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:00,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:00,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:00,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741944_1120 (size=12301) 2024-11-19T04:41:00,586 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/dae33271fb53418a9c05e3052c061216 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dae33271fb53418a9c05e3052c061216 2024-11-19T04:41:00,587 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/43131c2f478746d5bac4e6d64d6060d4 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/43131c2f478746d5bac4e6d64d6060d4 2024-11-19T04:41:00,591 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/A of e97938390deb1164b875a27699276aad into dae33271fb53418a9c05e3052c061216(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:00,591 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:41:00,591 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/A, priority=13, startTime=1731991260161; duration=0sec 2024-11-19T04:41:00,592 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:00,592 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:A 2024-11-19T04:41:00,592 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:00,592 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/B of e97938390deb1164b875a27699276aad into 43131c2f478746d5bac4e6d64d6060d4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:00,592 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:41:00,592 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/B, priority=13, startTime=1731991260161; duration=0sec 2024-11-19T04:41:00,592 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:00,592 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:B 2024-11-19T04:41:00,593 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:00,593 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): e97938390deb1164b875a27699276aad/C is initiating minor compaction (all files) 2024-11-19T04:41:00,593 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e97938390deb1164b875a27699276aad/C in TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:00,593 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c94e0d115b794ce682661aa5931ebe75, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b2bea343fbeb481b874821b799828f01, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dcb503fda8af4a5ba74d649838984698] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp, totalSize=37.1 K 2024-11-19T04:41:00,593 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94e0d115b794ce682661aa5931ebe75, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=492, earliestPutTs=1731991257490 2024-11-19T04:41:00,594 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2bea343fbeb481b874821b799828f01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731991257670 2024-11-19T04:41:00,594 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcb503fda8af4a5ba74d649838984698, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1731991258326 2024-11-19T04:41:00,602 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e97938390deb1164b875a27699276aad#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:00,602 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/3af1c723af074c6b8a5fe403e033fb15 is 50, key is test_row_0/C:col10/1731991258869/Put/seqid=0 2024-11-19T04:41:00,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741945_1121 (size=13493) 2024-11-19T04:41:00,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/1094c3bc63984f85937148807b20d5fb 2024-11-19T04:41:00,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/87bd99692c684abba743d030b525cc05 is 50, key is test_row_0/B:col10/1731991258940/Put/seqid=0 2024-11-19T04:41:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741946_1122 (size=12301) 2024-11-19T04:41:01,013 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/3af1c723af074c6b8a5fe403e033fb15 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3af1c723af074c6b8a5fe403e033fb15 2024-11-19T04:41:01,019 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e97938390deb1164b875a27699276aad/C of e97938390deb1164b875a27699276aad into 3af1c723af074c6b8a5fe403e033fb15(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:01,019 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:41:01,019 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., storeName=e97938390deb1164b875a27699276aad/C, priority=13, startTime=1731991260161; duration=0sec 2024-11-19T04:41:01,020 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:01,020 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e97938390deb1164b875a27699276aad:C 2024-11-19T04:41:01,111 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T04:41:01,111 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:01,111 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:01,111 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:01,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:01,118 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T04:41:01,118 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:01,118 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:01,119 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:01,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2587 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2513 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1176 2024-11-19T04:41:01,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3527 rows 2024-11-19T04:41:01,119 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:41:01,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:41:01,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:01,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:01,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:01,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T04:41:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T04:41:01,132 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991261131"}]},"ts":"1731991261131"} 2024-11-19T04:41:01,134 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T04:41:01,134 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T04:41:01,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:41:01,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, UNASSIGN}] 2024-11-19T04:41:01,142 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, UNASSIGN 2024-11-19T04:41:01,144 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=e97938390deb1164b875a27699276aad, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:01,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, UNASSIGN because future has completed 2024-11-19T04:41:01,147 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:41:01,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:41:01,218 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T04:41:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T04:41:01,304 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close e97938390deb1164b875a27699276aad 2024-11-19T04:41:01,304 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:41:01,305 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing e97938390deb1164b875a27699276aad, disabling compactions & flushes 2024-11-19T04:41:01,305 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:01,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/87bd99692c684abba743d030b525cc05 2024-11-19T04:41:01,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/dec7b341c85241d0b6496e96c8f52582 is 50, key is test_row_0/C:col10/1731991258940/Put/seqid=0 2024-11-19T04:41:01,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741947_1123 (size=12301) 2024-11-19T04:41:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T04:41:01,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/dec7b341c85241d0b6496e96c8f52582 2024-11-19T04:41:01,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/1094c3bc63984f85937148807b20d5fb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/1094c3bc63984f85937148807b20d5fb 2024-11-19T04:41:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T04:41:01,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/1094c3bc63984f85937148807b20d5fb, entries=150, sequenceid=547, filesize=12.0 K 2024-11-19T04:41:01,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/87bd99692c684abba743d030b525cc05 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/87bd99692c684abba743d030b525cc05 2024-11-19T04:41:01,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/87bd99692c684abba743d030b525cc05, entries=150, sequenceid=547, filesize=12.0 K 2024-11-19T04:41:01,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/dec7b341c85241d0b6496e96c8f52582 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dec7b341c85241d0b6496e96c8f52582 2024-11-19T04:41:01,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dec7b341c85241d0b6496e96c8f52582, entries=150, sequenceid=547, filesize=12.0 K 2024-11-19T04:41:01,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for e97938390deb1164b875a27699276aad in 1279ms, sequenceid=547, compaction requested=false 2024-11-19T04:41:01,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e97938390deb1164b875a27699276aad: 2024-11-19T04:41:01,777 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. after waiting 0 ms 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:01,777 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing e97938390deb1164b875a27699276aad 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=A 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=B 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e97938390deb1164b875a27699276aad, store=C 2024-11-19T04:41:01,777 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:01,786 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/6103ac4724964ef990fc2d829ab770ae is 50, key is test_row_0/A:col10/1731991261117/Put/seqid=0 2024-11-19T04:41:01,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741948_1124 (size=12301) 2024-11-19T04:41:02,199 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/6103ac4724964ef990fc2d829ab770ae 2024-11-19T04:41:02,212 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/8a30a7b261584fc4bd8057debc20bc54 is 50, key is test_row_0/B:col10/1731991261117/Put/seqid=0 2024-11-19T04:41:02,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741949_1125 (size=12301) 2024-11-19T04:41:02,218 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/8a30a7b261584fc4bd8057debc20bc54 2024-11-19T04:41:02,226 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/fdb4c71b6ae346b7ac2b6df6198578dd is 50, key is test_row_0/C:col10/1731991261117/Put/seqid=0 2024-11-19T04:41:02,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741950_1126 (size=12301) 2024-11-19T04:41:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T04:41:02,631 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/fdb4c71b6ae346b7ac2b6df6198578dd 2024-11-19T04:41:02,637 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/A/6103ac4724964ef990fc2d829ab770ae as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6103ac4724964ef990fc2d829ab770ae 2024-11-19T04:41:02,642 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6103ac4724964ef990fc2d829ab770ae, entries=150, sequenceid=557, filesize=12.0 K 2024-11-19T04:41:02,643 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/B/8a30a7b261584fc4bd8057debc20bc54 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/8a30a7b261584fc4bd8057debc20bc54 2024-11-19T04:41:02,647 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/8a30a7b261584fc4bd8057debc20bc54, entries=150, sequenceid=557, filesize=12.0 K 2024-11-19T04:41:02,648 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/.tmp/C/fdb4c71b6ae346b7ac2b6df6198578dd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/fdb4c71b6ae346b7ac2b6df6198578dd 2024-11-19T04:41:02,653 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/fdb4c71b6ae346b7ac2b6df6198578dd, entries=150, sequenceid=557, filesize=12.0 K 2024-11-19T04:41:02,654 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e97938390deb1164b875a27699276aad in 877ms, sequenceid=557, compaction requested=true 2024-11-19T04:41:02,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2fea7607fa0c4c88bec9112863f18d11, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3da1fc79b8045b691c76b36eda77dfb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/309af6ddd91c4e03981e858628e36dd1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e3bddb51aece44f7b21af1fc10c67d99, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/77c4870f2a6d483a90bfaf95912b1973, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8650e133f1a94283a6eeae2b75ba7061, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/b510b14d914e47efbce5958b5f2f787b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/9fc515991a734c8697b2f3347e2287e9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/71c1b916f95d49c3a1e6053327896245, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e0b3402b677f420b950daacd4d94420a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8887c93f441b426e8ff06b97d6a4a8bc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c67d86886f254fd2970391712935b59f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ab1714882de14389bbd34cb6ccb9df61, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3de997bfded428ab235bbb0960e2889, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/05ddd007bc3e4976aaed8a1aedfc4499, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6d34c8e48e8047c2a1fb926805d4259d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/fd74a6f720a041e9890997294c59f1e8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/14d327dd330548b3887682c3b454a020, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/a8d3b13abd4b4f1da2f454449c363243, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/cd49157e7aa949d19e06bbe329ca9b1e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e85b34feb1d447e098c9c4a22683b231, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/54ba1be92c674738b0e4ca1a9a4a7c38, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2bc4b51bd377411ea0f4561b127f71dd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/d28eedaa40a24e5183b25ea65a4c115a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ba1a366c7e454d338443405e6e394991, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/bb474f6c7bdb49318e1d063dad3135e5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/549acfee402e4188888c63ebf63fa36c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/26fb3e6090304eaa9348855fcbb94090, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/87570a0c5fe94ea58de92bbdefa795a4, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/5c6016378dd649779eaa283ebb14b89e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/4cbd6105777a4eea8574ecf2ac75f280, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dc88598c1f504ff2a3bebd65d24b8745, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e1e00ff88a444588948bcb3e6ce09163, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/114ba3b66189434492e455754e3b6660] to archive 2024-11-19T04:41:02,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:41:02,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2fea7607fa0c4c88bec9112863f18d11 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2fea7607fa0c4c88bec9112863f18d11 2024-11-19T04:41:02,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3da1fc79b8045b691c76b36eda77dfb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3da1fc79b8045b691c76b36eda77dfb 2024-11-19T04:41:02,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/309af6ddd91c4e03981e858628e36dd1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/309af6ddd91c4e03981e858628e36dd1 2024-11-19T04:41:02,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e3bddb51aece44f7b21af1fc10c67d99 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e3bddb51aece44f7b21af1fc10c67d99 2024-11-19T04:41:02,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/77c4870f2a6d483a90bfaf95912b1973 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/77c4870f2a6d483a90bfaf95912b1973 2024-11-19T04:41:02,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8a1e7f631c7243e2a1c3b6f54ffc0b4e 2024-11-19T04:41:02,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8650e133f1a94283a6eeae2b75ba7061 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8650e133f1a94283a6eeae2b75ba7061 2024-11-19T04:41:02,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/b510b14d914e47efbce5958b5f2f787b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/b510b14d914e47efbce5958b5f2f787b 2024-11-19T04:41:02,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/9fc515991a734c8697b2f3347e2287e9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/9fc515991a734c8697b2f3347e2287e9 2024-11-19T04:41:02,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/71c1b916f95d49c3a1e6053327896245 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/71c1b916f95d49c3a1e6053327896245 2024-11-19T04:41:02,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e0b3402b677f420b950daacd4d94420a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e0b3402b677f420b950daacd4d94420a 2024-11-19T04:41:02,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8887c93f441b426e8ff06b97d6a4a8bc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/8887c93f441b426e8ff06b97d6a4a8bc 2024-11-19T04:41:02,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c67d86886f254fd2970391712935b59f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c67d86886f254fd2970391712935b59f 2024-11-19T04:41:02,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ab1714882de14389bbd34cb6ccb9df61 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ab1714882de14389bbd34cb6ccb9df61 2024-11-19T04:41:02,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3de997bfded428ab235bbb0960e2889 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/c3de997bfded428ab235bbb0960e2889 2024-11-19T04:41:02,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/05ddd007bc3e4976aaed8a1aedfc4499 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/05ddd007bc3e4976aaed8a1aedfc4499 2024-11-19T04:41:02,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6d34c8e48e8047c2a1fb926805d4259d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6d34c8e48e8047c2a1fb926805d4259d 2024-11-19T04:41:02,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/fd74a6f720a041e9890997294c59f1e8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/fd74a6f720a041e9890997294c59f1e8 2024-11-19T04:41:02,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/14d327dd330548b3887682c3b454a020 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/14d327dd330548b3887682c3b454a020 2024-11-19T04:41:02,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/a8d3b13abd4b4f1da2f454449c363243 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/a8d3b13abd4b4f1da2f454449c363243 2024-11-19T04:41:02,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/cd49157e7aa949d19e06bbe329ca9b1e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/cd49157e7aa949d19e06bbe329ca9b1e 2024-11-19T04:41:02,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e85b34feb1d447e098c9c4a22683b231 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e85b34feb1d447e098c9c4a22683b231 2024-11-19T04:41:02,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/54ba1be92c674738b0e4ca1a9a4a7c38 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/54ba1be92c674738b0e4ca1a9a4a7c38 2024-11-19T04:41:02,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2bc4b51bd377411ea0f4561b127f71dd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/2bc4b51bd377411ea0f4561b127f71dd 2024-11-19T04:41:02,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/d28eedaa40a24e5183b25ea65a4c115a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/d28eedaa40a24e5183b25ea65a4c115a 2024-11-19T04:41:02,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ba1a366c7e454d338443405e6e394991 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/ba1a366c7e454d338443405e6e394991 2024-11-19T04:41:02,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/bb474f6c7bdb49318e1d063dad3135e5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/bb474f6c7bdb49318e1d063dad3135e5 2024-11-19T04:41:02,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/549acfee402e4188888c63ebf63fa36c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/549acfee402e4188888c63ebf63fa36c 2024-11-19T04:41:02,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/26fb3e6090304eaa9348855fcbb94090 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/26fb3e6090304eaa9348855fcbb94090 2024-11-19T04:41:02,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/87570a0c5fe94ea58de92bbdefa795a4 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/87570a0c5fe94ea58de92bbdefa795a4 2024-11-19T04:41:02,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/5c6016378dd649779eaa283ebb14b89e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/5c6016378dd649779eaa283ebb14b89e 2024-11-19T04:41:02,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/4cbd6105777a4eea8574ecf2ac75f280 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/4cbd6105777a4eea8574ecf2ac75f280 2024-11-19T04:41:02,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dc88598c1f504ff2a3bebd65d24b8745 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dc88598c1f504ff2a3bebd65d24b8745 2024-11-19T04:41:02,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e1e00ff88a444588948bcb3e6ce09163 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/e1e00ff88a444588948bcb3e6ce09163 2024-11-19T04:41:02,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/114ba3b66189434492e455754e3b6660 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/114ba3b66189434492e455754e3b6660 2024-11-19T04:41:02,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e2e740689cef47d1b036711ccfd76bba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b64b5d3bc52a41b9888704e3f9d01927, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/475ca57c22ed4005a84b2949fcb5fcc9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/c2b6cd4d8423443c87252f09c6dbc0f8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1a4230fc2bb84e0fa14751c08d4b694a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/95705875648a493db45b1f190b272b7e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/9072f45211564756bb29892d65880a02, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/6dc9301006364d3d9c7b9b8c35b6dbca, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b9a003b3280a480daf965285d4472d1a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/33f063389a3b48a5be036725f17bab71, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a944a5cf8b8f4e7b87fad8eeff82ea4c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ebe315594c0f44009d1c9c1c91982fa7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fb6698c2d4964cf0954612a36e6e3689, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/11a9059129a24d55acb818a5fe48b5a6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/60dc98ffb50d416e816f68f64c92e7f5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fdb9f2e30e8d4f87a0e5054632001eb9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/2c9bfc72afbf47d7ad66d684c234e391, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e6ef3b16c4d54805aa7396d93ef706d8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a698561ad8f6461591457bc6301c96fb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ed6a5536454647d2868d5a69ff37a5f2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b66229db26594282a459d4c23c80cbc0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/65d48d458578450992647dae1bc03a62, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1cea150a91b24f14833d096da90795ea, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e68fb8e53f484f87bd7d1220be2573ce, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/68256b52df7f48a49bac429f2e1fdd00, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/7396ceb465104df09898c5056440fb4d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/23c9b4ea187545abb417a7144609615b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/f98b420fb41f4404a5bb240265c79efc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/0a1b58c2f671431d993f26c8305b1443, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/5c56317c616d4c72ac8a156282923b58, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/eccdf45d729441cf8cbf4a2540589524, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/66b5dea6478f4c5a920ecaf93c4ebb4f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/03b9b376f4744b5fb959f433135e0517, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/d5912087d6184d678db14e8252df845f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/35d63e558ae141468e95736681b53247] to archive 2024-11-19T04:41:02,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:41:02,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e2e740689cef47d1b036711ccfd76bba to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e2e740689cef47d1b036711ccfd76bba 2024-11-19T04:41:02,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b64b5d3bc52a41b9888704e3f9d01927 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b64b5d3bc52a41b9888704e3f9d01927 2024-11-19T04:41:02,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/475ca57c22ed4005a84b2949fcb5fcc9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/475ca57c22ed4005a84b2949fcb5fcc9 2024-11-19T04:41:02,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/c2b6cd4d8423443c87252f09c6dbc0f8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/c2b6cd4d8423443c87252f09c6dbc0f8 2024-11-19T04:41:02,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1a4230fc2bb84e0fa14751c08d4b694a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1a4230fc2bb84e0fa14751c08d4b694a 2024-11-19T04:41:02,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/95705875648a493db45b1f190b272b7e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/95705875648a493db45b1f190b272b7e 2024-11-19T04:41:02,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/9072f45211564756bb29892d65880a02 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/9072f45211564756bb29892d65880a02 2024-11-19T04:41:02,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/6dc9301006364d3d9c7b9b8c35b6dbca to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/6dc9301006364d3d9c7b9b8c35b6dbca 2024-11-19T04:41:02,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b9a003b3280a480daf965285d4472d1a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b9a003b3280a480daf965285d4472d1a 2024-11-19T04:41:02,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/33f063389a3b48a5be036725f17bab71 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/33f063389a3b48a5be036725f17bab71 2024-11-19T04:41:02,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a944a5cf8b8f4e7b87fad8eeff82ea4c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a944a5cf8b8f4e7b87fad8eeff82ea4c 2024-11-19T04:41:02,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ebe315594c0f44009d1c9c1c91982fa7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ebe315594c0f44009d1c9c1c91982fa7 2024-11-19T04:41:02,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fb6698c2d4964cf0954612a36e6e3689 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fb6698c2d4964cf0954612a36e6e3689 2024-11-19T04:41:02,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/11a9059129a24d55acb818a5fe48b5a6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/11a9059129a24d55acb818a5fe48b5a6 2024-11-19T04:41:02,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/60dc98ffb50d416e816f68f64c92e7f5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/60dc98ffb50d416e816f68f64c92e7f5 2024-11-19T04:41:02,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fdb9f2e30e8d4f87a0e5054632001eb9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/fdb9f2e30e8d4f87a0e5054632001eb9 2024-11-19T04:41:02,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/2c9bfc72afbf47d7ad66d684c234e391 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/2c9bfc72afbf47d7ad66d684c234e391 2024-11-19T04:41:02,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e6ef3b16c4d54805aa7396d93ef706d8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e6ef3b16c4d54805aa7396d93ef706d8 2024-11-19T04:41:02,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a698561ad8f6461591457bc6301c96fb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/a698561ad8f6461591457bc6301c96fb 2024-11-19T04:41:02,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ed6a5536454647d2868d5a69ff37a5f2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/ed6a5536454647d2868d5a69ff37a5f2 2024-11-19T04:41:02,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b66229db26594282a459d4c23c80cbc0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/b66229db26594282a459d4c23c80cbc0 2024-11-19T04:41:02,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/65d48d458578450992647dae1bc03a62 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/65d48d458578450992647dae1bc03a62 2024-11-19T04:41:02,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1cea150a91b24f14833d096da90795ea to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/1cea150a91b24f14833d096da90795ea 2024-11-19T04:41:02,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e68fb8e53f484f87bd7d1220be2573ce to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/e68fb8e53f484f87bd7d1220be2573ce 2024-11-19T04:41:02,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/68256b52df7f48a49bac429f2e1fdd00 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/68256b52df7f48a49bac429f2e1fdd00 2024-11-19T04:41:02,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/7396ceb465104df09898c5056440fb4d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/7396ceb465104df09898c5056440fb4d 2024-11-19T04:41:02,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/23c9b4ea187545abb417a7144609615b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/23c9b4ea187545abb417a7144609615b 2024-11-19T04:41:02,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/f98b420fb41f4404a5bb240265c79efc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/f98b420fb41f4404a5bb240265c79efc 2024-11-19T04:41:02,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/0a1b58c2f671431d993f26c8305b1443 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/0a1b58c2f671431d993f26c8305b1443 2024-11-19T04:41:02,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/5c56317c616d4c72ac8a156282923b58 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/5c56317c616d4c72ac8a156282923b58 2024-11-19T04:41:02,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/eccdf45d729441cf8cbf4a2540589524 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/eccdf45d729441cf8cbf4a2540589524 2024-11-19T04:41:02,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/66b5dea6478f4c5a920ecaf93c4ebb4f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/66b5dea6478f4c5a920ecaf93c4ebb4f 2024-11-19T04:41:02,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/03b9b376f4744b5fb959f433135e0517 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/03b9b376f4744b5fb959f433135e0517 2024-11-19T04:41:02,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/d5912087d6184d678db14e8252df845f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/d5912087d6184d678db14e8252df845f 2024-11-19T04:41:02,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/35d63e558ae141468e95736681b53247 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/35d63e558ae141468e95736681b53247 2024-11-19T04:41:02,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b477cc5a8578479ca3ddf7999957fd6a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e07d76691512480384ceee54dc78cf2b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/0f4514657094434a908533e4085f9f56, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/64acfff3ef464d5f947a5398eac893c3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/66a8cfbb42ef475f94e582dc6226240b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e530d8a00f9c44cf9f470b323cd4dea3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9361f5513c124a1fbebe0535cfbfc0fb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/d081b4065af24e7e9da5a82c441148a8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/7ed9c2ad30df41afa17ceab33190a916, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/49bfee3dd72d464abb8ddaf47831a17b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/725a422c7e2540d883a14ed3e6ee18e6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c398227c260847e4a1eac23edf36ceef, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/cc0a759d0d3047bbabd89820f998c671, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f3ac4e0a337e49a8820b6975f45e04ae, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9b430b0137554065965c436bc1928062, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8e7479ff306b4ae4b70f8ecb2f7a2439, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/28d6fe25f2984fca85dc04c12d385d1f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f6fb404d5d73438fae4346d11c2c288f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/05a38fbd5d6848dc92b686bc6e9ca49a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9825540f42ae426c97f37c90dec5f424, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8f0c03b94a534f2d958f602784b8fcbe, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/46f635d7c34c48bdb9dd92b82d6e7671, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/1b200d6544564e48bd8e89c8615126e2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f9ad5e76c11d4a69a38facf6f14eeedc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/6906a0534ad74649ba3be36516dfbffb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/16a06c597dbc444fb3fe3fcbcd28c8be, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9cb9962b80ab4a9ba96aede9ef06f89a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9f649c15af734774b79153de59a148bb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a6b2a4765bef4dd1bc551f34a51f7c05, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/252add34272a4346a0b3eacac808b657, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3a2d1fa66c6c4d55ad2c5a05d5131af3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c94e0d115b794ce682661aa5931ebe75, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a69dfd165f25487889ce01cc5356cc8b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b2bea343fbeb481b874821b799828f01, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dcb503fda8af4a5ba74d649838984698] to archive 2024-11-19T04:41:02,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:41:02,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b477cc5a8578479ca3ddf7999957fd6a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b477cc5a8578479ca3ddf7999957fd6a 2024-11-19T04:41:02,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e07d76691512480384ceee54dc78cf2b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e07d76691512480384ceee54dc78cf2b 2024-11-19T04:41:02,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/0f4514657094434a908533e4085f9f56 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/0f4514657094434a908533e4085f9f56 2024-11-19T04:41:02,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/64acfff3ef464d5f947a5398eac893c3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/64acfff3ef464d5f947a5398eac893c3 2024-11-19T04:41:02,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/66a8cfbb42ef475f94e582dc6226240b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/66a8cfbb42ef475f94e582dc6226240b 2024-11-19T04:41:02,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e530d8a00f9c44cf9f470b323cd4dea3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/e530d8a00f9c44cf9f470b323cd4dea3 2024-11-19T04:41:02,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9361f5513c124a1fbebe0535cfbfc0fb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9361f5513c124a1fbebe0535cfbfc0fb 2024-11-19T04:41:02,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/d081b4065af24e7e9da5a82c441148a8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/d081b4065af24e7e9da5a82c441148a8 2024-11-19T04:41:02,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/7ed9c2ad30df41afa17ceab33190a916 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/7ed9c2ad30df41afa17ceab33190a916 2024-11-19T04:41:02,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/49bfee3dd72d464abb8ddaf47831a17b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/49bfee3dd72d464abb8ddaf47831a17b 2024-11-19T04:41:02,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/725a422c7e2540d883a14ed3e6ee18e6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/725a422c7e2540d883a14ed3e6ee18e6 2024-11-19T04:41:02,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c398227c260847e4a1eac23edf36ceef to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c398227c260847e4a1eac23edf36ceef 2024-11-19T04:41:02,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/cc0a759d0d3047bbabd89820f998c671 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/cc0a759d0d3047bbabd89820f998c671 2024-11-19T04:41:02,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f3ac4e0a337e49a8820b6975f45e04ae to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f3ac4e0a337e49a8820b6975f45e04ae 2024-11-19T04:41:02,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9b430b0137554065965c436bc1928062 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9b430b0137554065965c436bc1928062 2024-11-19T04:41:02,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8e7479ff306b4ae4b70f8ecb2f7a2439 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8e7479ff306b4ae4b70f8ecb2f7a2439 2024-11-19T04:41:02,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/28d6fe25f2984fca85dc04c12d385d1f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/28d6fe25f2984fca85dc04c12d385d1f 2024-11-19T04:41:02,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f6fb404d5d73438fae4346d11c2c288f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f6fb404d5d73438fae4346d11c2c288f 2024-11-19T04:41:02,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/05a38fbd5d6848dc92b686bc6e9ca49a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/05a38fbd5d6848dc92b686bc6e9ca49a 2024-11-19T04:41:02,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9825540f42ae426c97f37c90dec5f424 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9825540f42ae426c97f37c90dec5f424 2024-11-19T04:41:02,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8f0c03b94a534f2d958f602784b8fcbe to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/8f0c03b94a534f2d958f602784b8fcbe 2024-11-19T04:41:02,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/46f635d7c34c48bdb9dd92b82d6e7671 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/46f635d7c34c48bdb9dd92b82d6e7671 2024-11-19T04:41:02,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/1b200d6544564e48bd8e89c8615126e2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/1b200d6544564e48bd8e89c8615126e2 2024-11-19T04:41:02,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f9ad5e76c11d4a69a38facf6f14eeedc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/f9ad5e76c11d4a69a38facf6f14eeedc 2024-11-19T04:41:02,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/6906a0534ad74649ba3be36516dfbffb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/6906a0534ad74649ba3be36516dfbffb 2024-11-19T04:41:02,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/16a06c597dbc444fb3fe3fcbcd28c8be to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/16a06c597dbc444fb3fe3fcbcd28c8be 2024-11-19T04:41:02,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9cb9962b80ab4a9ba96aede9ef06f89a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9cb9962b80ab4a9ba96aede9ef06f89a 2024-11-19T04:41:02,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9f649c15af734774b79153de59a148bb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/9f649c15af734774b79153de59a148bb 2024-11-19T04:41:02,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a6b2a4765bef4dd1bc551f34a51f7c05 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a6b2a4765bef4dd1bc551f34a51f7c05 2024-11-19T04:41:02,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/252add34272a4346a0b3eacac808b657 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/252add34272a4346a0b3eacac808b657 2024-11-19T04:41:02,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3a2d1fa66c6c4d55ad2c5a05d5131af3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3a2d1fa66c6c4d55ad2c5a05d5131af3 2024-11-19T04:41:02,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c94e0d115b794ce682661aa5931ebe75 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/c94e0d115b794ce682661aa5931ebe75 2024-11-19T04:41:02,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a69dfd165f25487889ce01cc5356cc8b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/a69dfd165f25487889ce01cc5356cc8b 2024-11-19T04:41:02,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b2bea343fbeb481b874821b799828f01 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/b2bea343fbeb481b874821b799828f01 2024-11-19T04:41:02,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dcb503fda8af4a5ba74d649838984698 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dcb503fda8af4a5ba74d649838984698 2024-11-19T04:41:02,885 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/recovered.edits/560.seqid, newMaxSeqId=560, maxSeqId=1 2024-11-19T04:41:02,890 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. 2024-11-19T04:41:02,890 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for e97938390deb1164b875a27699276aad: Waiting for close lock at 1731991261305Running coprocessor pre-close hooks at 1731991261305Disabling compacts and flushes for region at 1731991261305Disabling writes for close at 1731991261777 (+472 ms)Obtaining lock to block concurrent updates at 1731991261777Preparing flush snapshotting stores in e97938390deb1164b875a27699276aad at 1731991261777Finished memstore snapshotting TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731991261777Flushing stores of TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad. at 1731991261778 (+1 ms)Flushing e97938390deb1164b875a27699276aad/A: creating writer at 1731991261778Flushing e97938390deb1164b875a27699276aad/A: appending metadata at 1731991261785 (+7 ms)Flushing e97938390deb1164b875a27699276aad/A: closing flushed file at 1731991261785Flushing e97938390deb1164b875a27699276aad/B: creating writer at 1731991262205 (+420 ms)Flushing e97938390deb1164b875a27699276aad/B: appending metadata at 1731991262211 (+6 ms)Flushing e97938390deb1164b875a27699276aad/B: closing flushed file at 1731991262211Flushing e97938390deb1164b875a27699276aad/C: creating writer at 1731991262223 (+12 ms)Flushing e97938390deb1164b875a27699276aad/C: appending metadata at 1731991262225 (+2 ms)Flushing e97938390deb1164b875a27699276aad/C: closing flushed file at 1731991262225Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b7ade49: reopening flushed file at 1731991262636 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7638be42: reopening flushed file at 1731991262642 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49d6fad: reopening flushed file at 1731991262648 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e97938390deb1164b875a27699276aad in 877ms, sequenceid=557, compaction requested=true at 1731991262654 (+6 ms)Writing region close event to WAL at 1731991262860 (+206 ms)Running coprocessor post-close hooks at 1731991262887 (+27 ms)Closed at 1731991262890 (+3 ms) 2024-11-19T04:41:02,893 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed e97938390deb1164b875a27699276aad 2024-11-19T04:41:02,894 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=e97938390deb1164b875a27699276aad, regionState=CLOSED 2024-11-19T04:41:02,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:41:02,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T04:41:02,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure e97938390deb1164b875a27699276aad, server=08a7f35e60d4,32975,1731991233019 in 1.7500 sec 2024-11-19T04:41:02,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T04:41:02,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e97938390deb1164b875a27699276aad, UNASSIGN in 1.7590 sec 2024-11-19T04:41:02,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T04:41:02,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7710 sec 2024-11-19T04:41:02,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991262912"}]},"ts":"1731991262912"} 2024-11-19T04:41:02,914 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T04:41:02,914 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T04:41:02,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7890 sec 2024-11-19T04:41:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T04:41:03,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:03,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T04:41:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,289 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,291 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T04:41:03,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad 2024-11-19T04:41:03,313 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/recovered.edits] 2024-11-19T04:41:03,327 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/1094c3bc63984f85937148807b20d5fb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/1094c3bc63984f85937148807b20d5fb 2024-11-19T04:41:03,330 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6103ac4724964ef990fc2d829ab770ae to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/6103ac4724964ef990fc2d829ab770ae 2024-11-19T04:41:03,332 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dae33271fb53418a9c05e3052c061216 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/A/dae33271fb53418a9c05e3052c061216 2024-11-19T04:41:03,337 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/43131c2f478746d5bac4e6d64d6060d4 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/43131c2f478746d5bac4e6d64d6060d4 2024-11-19T04:41:03,339 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/87bd99692c684abba743d030b525cc05 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/87bd99692c684abba743d030b525cc05 2024-11-19T04:41:03,341 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/8a30a7b261584fc4bd8057debc20bc54 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/B/8a30a7b261584fc4bd8057debc20bc54 2024-11-19T04:41:03,345 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3af1c723af074c6b8a5fe403e033fb15 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/3af1c723af074c6b8a5fe403e033fb15 2024-11-19T04:41:03,346 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dec7b341c85241d0b6496e96c8f52582 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/dec7b341c85241d0b6496e96c8f52582 2024-11-19T04:41:03,348 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/fdb4c71b6ae346b7ac2b6df6198578dd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/C/fdb4c71b6ae346b7ac2b6df6198578dd 2024-11-19T04:41:03,352 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/recovered.edits/560.seqid to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad/recovered.edits/560.seqid 2024-11-19T04:41:03,353 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/e97938390deb1164b875a27699276aad 2024-11-19T04:41:03,354 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T04:41:03,360 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T04:41:03,373 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T04:41:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T04:41:03,421 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T04:41:03,424 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,424 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T04:41:03,424 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731991263424"}]},"ts":"9223372036854775807"} 2024-11-19T04:41:03,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T04:41:03,431 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e97938390deb1164b875a27699276aad, NAME => 'TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T04:41:03,431 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T04:41:03,431 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731991263431"}]},"ts":"9223372036854775807"} 2024-11-19T04:41:03,442 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T04:41:03,443 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 160 msec 2024-11-19T04:41:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-19T04:41:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T04:41:03,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:03,623 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;08a7f35e60d4:32975-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_865084663_22 at /127.0.0.1:38692 [Waiting for operation #359] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_357640213_22 at /127.0.0.1:38810 [Waiting for operation #345] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 317) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12321 (was 12884) 2024-11-19T04:41:03,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=12321 2024-11-19T04:41:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:41:03,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:41:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:03,636 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T04:41:03,637 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:03,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-19T04:41:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T04:41:03,637 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T04:41:03,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741951_1127 (size=1061) 2024-11-19T04:41:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T04:41:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T04:41:04,047 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:41:04,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741952_1128 (size=53) 2024-11-19T04:41:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T04:41:04,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:41:04,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2579823445e39a7eaea2aba4ffb4cf67, disabling compactions & flushes 2024-11-19T04:41:04,461 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. after waiting 0 ms 2024-11-19T04:41:04,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,461 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2579823445e39a7eaea2aba4ffb4cf67: Waiting for close lock at 1731991264461Disabling compacts and flushes for region at 1731991264461Disabling writes for close at 1731991264461Writing region close event to WAL at 1731991264461Closed at 1731991264461 2024-11-19T04:41:04,463 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T04:41:04,464 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731991264463"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731991264463"}]},"ts":"1731991264463"} 2024-11-19T04:41:04,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T04:41:04,469 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T04:41:04,469 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991264469"}]},"ts":"1731991264469"} 2024-11-19T04:41:04,472 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T04:41:04,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, ASSIGN}] 2024-11-19T04:41:04,474 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, ASSIGN 2024-11-19T04:41:04,481 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, ASSIGN; state=OFFLINE, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=false 2024-11-19T04:41:04,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:04,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, ASSIGN because future has completed 2024-11-19T04:41:04,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:41:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T04:41:04,790 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,790 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:41:04,791 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,791 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:41:04,791 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,791 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,792 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,793 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:04,794 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2579823445e39a7eaea2aba4ffb4cf67 columnFamilyName A 2024-11-19T04:41:04,794 DEBUG [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:04,795 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(327): Store=2579823445e39a7eaea2aba4ffb4cf67/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:04,795 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,796 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:04,796 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2579823445e39a7eaea2aba4ffb4cf67 columnFamilyName B 2024-11-19T04:41:04,796 DEBUG [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:04,797 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(327): Store=2579823445e39a7eaea2aba4ffb4cf67/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:04,797 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,797 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:04,798 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2579823445e39a7eaea2aba4ffb4cf67 columnFamilyName C 2024-11-19T04:41:04,798 DEBUG [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:04,798 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(327): Store=2579823445e39a7eaea2aba4ffb4cf67/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:04,798 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,798 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,799 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,799 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,800 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,800 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,801 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:41:04,802 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,803 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:41:04,804 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 2579823445e39a7eaea2aba4ffb4cf67; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72759341, jitterRate=0.08419866859912872}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:41:04,804 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:04,804 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 2579823445e39a7eaea2aba4ffb4cf67: Running coprocessor pre-open hook at 1731991264791Writing region info on filesystem at 1731991264791Initializing all the Stores at 1731991264792 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991264792Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991264792Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991264792Cleaning up temporary data from old regions at 1731991264800 (+8 ms)Running coprocessor post-open hooks at 1731991264804 (+4 ms)Region opened successfully at 1731991264804 2024-11-19T04:41:04,805 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., pid=36, masterSystemTime=1731991264787 2024-11-19T04:41:04,807 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,807 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:04,808 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:04,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:41:04,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-19T04:41:04,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 in 177 msec 2024-11-19T04:41:04,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-19T04:41:04,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, ASSIGN in 339 msec 2024-11-19T04:41:04,815 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T04:41:04,815 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991264815"}]},"ts":"1731991264815"} 2024-11-19T04:41:04,817 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T04:41:04,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T04:41:04,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-19T04:41:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T04:41:05,787 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:05,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:05,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:05,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:05,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d015c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:05,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8cfaec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:05,793 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:05,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:05,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:05,796 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:41:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:41:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5eaae187 2024-11-19T04:41:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:41:05,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:41:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:41:05,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:41:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:05,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741953_1129 (size=1097) 2024-11-19T04:41:06,225 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T04:41:06,225 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T04:41:06,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:41:06,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, REOPEN/MOVE}] 2024-11-19T04:41:06,237 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, REOPEN/MOVE 2024-11-19T04:41:06,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:06,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, REOPEN/MOVE because future has completed 2024-11-19T04:41:06,241 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:41:06,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:41:06,396 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,396 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:41:06,397 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 2579823445e39a7eaea2aba4ffb4cf67, disabling compactions & flushes 2024-11-19T04:41:06,397 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,397 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,397 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. after waiting 0 ms 2024-11-19T04:41:06,397 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,402 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T04:41:06,402 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,402 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 2579823445e39a7eaea2aba4ffb4cf67: Waiting for close lock at 1731991266397Running coprocessor pre-close hooks at 1731991266397Disabling compacts and flushes for region at 1731991266397Disabling writes for close at 1731991266397Writing region close event to WAL at 1731991266397Running coprocessor post-close hooks at 1731991266402 (+5 ms)Closed at 1731991266402 2024-11-19T04:41:06,403 WARN [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 2579823445e39a7eaea2aba4ffb4cf67 to self. 2024-11-19T04:41:06,405 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,406 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=CLOSED 2024-11-19T04:41:06,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:41:06,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-19T04:41:06,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 in 168 msec 2024-11-19T04:41:06,412 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, REOPEN/MOVE; state=CLOSED, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=true 2024-11-19T04:41:06,563 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:06,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, REOPEN/MOVE because future has completed 2024-11-19T04:41:06,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:41:06,721 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,721 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:41:06,721 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,721 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:41:06,722 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,722 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,725 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,725 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:06,732 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2579823445e39a7eaea2aba4ffb4cf67 columnFamilyName A 2024-11-19T04:41:06,734 DEBUG [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:06,735 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(327): Store=2579823445e39a7eaea2aba4ffb4cf67/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:06,736 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,736 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:06,737 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2579823445e39a7eaea2aba4ffb4cf67 columnFamilyName B 2024-11-19T04:41:06,737 DEBUG [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:06,738 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(327): Store=2579823445e39a7eaea2aba4ffb4cf67/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:06,738 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,738 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:06,739 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2579823445e39a7eaea2aba4ffb4cf67 columnFamilyName C 2024-11-19T04:41:06,739 DEBUG [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:06,740 INFO [StoreOpener-2579823445e39a7eaea2aba4ffb4cf67-1 {}] regionserver.HStore(327): Store=2579823445e39a7eaea2aba4ffb4cf67/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:06,740 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,740 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,741 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,742 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,743 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,743 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,744 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:41:06,745 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,746 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 2579823445e39a7eaea2aba4ffb4cf67; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63576972, jitterRate=-0.052629292011260986}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:41:06,746 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,747 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 2579823445e39a7eaea2aba4ffb4cf67: Running coprocessor pre-open hook at 1731991266722Writing region info on filesystem at 1731991266722Initializing all the Stores at 1731991266723 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991266723Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991266723Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991266723Cleaning up temporary data from old regions at 1731991266743 (+20 ms)Running coprocessor post-open hooks at 1731991266746 (+3 ms)Region opened successfully at 1731991266747 (+1 ms) 2024-11-19T04:41:06,750 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., pid=41, masterSystemTime=1731991266716 2024-11-19T04:41:06,752 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,753 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,753 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=OPEN, openSeqNum=5, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:06,755 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46599 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=08a7f35e60d4,32975,1731991233019, table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T04:41:06,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:41:06,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-19T04:41:06,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 in 192 msec 2024-11-19T04:41:06,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-19T04:41:06,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, REOPEN/MOVE in 524 msec 2024-11-19T04:41:06,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-19T04:41:06,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-19T04:41:06,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 961 msec 2024-11-19T04:41:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-19T04:41:06,772 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:06,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10aa399c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,778 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,781 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,782 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb6cb38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,785 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ef276d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,787 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aefd27f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0a5cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b25578e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,800 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322c6b9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,802 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8d111b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,805 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa0baba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219c3a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,822 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b1146ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424ecd44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61833c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70104d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:06,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:06,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113608a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:06,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:06,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:06,833 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,834 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,834 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:06,835 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T04:41:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T04:41:06,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,839 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:06,840 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:06,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,845 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,846 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,847 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,861 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:06,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:41:06,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:06,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:06,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:06,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:06,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:06,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:06,881 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:06,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:41:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T04:41:06,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991266939, max timestamp=1731991266950Num uniques 859; 2024-11-19T04:41:06,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:06,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111990646900a4414b85b0e1cc64ab363eef_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991266858/Put/seqid=0 2024-11-19T04:41:06,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991266939, max timestamp=1731991266950Num uniques 859; 2024-11-19T04:41:06,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:06,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991266939, max timestamp=1731991266950Num uniques 859; 2024-11-19T04:41:06,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:06,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741954_1130 (size=17034) 2024-11-19T04:41:06,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T04:41:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:06,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:06,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:06,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:07,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111990646900a4414b85b0e1cc64ab363eef_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111990646900a4414b85b0e1cc64ab363eef_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:07,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/f95f32bb007b4ef49ddf9f42844a2a11, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:07,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/f95f32bb007b4ef49ddf9f42844a2a11 is 175, key is test_row_0/A:col10/1731991266858/Put/seqid=0 2024-11-19T04:41:07,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991327010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991327014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991327029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991327031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991327043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741955_1131 (size=48139) 2024-11-19T04:41:07,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/f95f32bb007b4ef49ddf9f42844a2a11 2024-11-19T04:41:07,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1cdd5a5d2d084fbeaab3d9c741d57d65 is 50, key is test_row_0/B:col10/1731991266858/Put/seqid=0 2024-11-19T04:41:07,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741956_1132 (size=12001) 2024-11-19T04:41:07,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T04:41:07,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:07,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1cdd5a5d2d084fbeaab3d9c741d57d65 2024-11-19T04:41:07,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:07,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:07,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:07,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T04:41:07,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991327154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991327159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991327159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991327159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991327159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b873f2f74d9144109417dbd08851beff is 50, key is test_row_0/C:col10/1731991266858/Put/seqid=0 2024-11-19T04:41:07,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741957_1133 (size=12001) 2024-11-19T04:41:07,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b873f2f74d9144109417dbd08851beff 2024-11-19T04:41:07,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/f95f32bb007b4ef49ddf9f42844a2a11 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11 2024-11-19T04:41:07,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11, entries=250, sequenceid=15, filesize=47.0 K 2024-11-19T04:41:07,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1cdd5a5d2d084fbeaab3d9c741d57d65 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1cdd5a5d2d084fbeaab3d9c741d57d65 2024-11-19T04:41:07,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1cdd5a5d2d084fbeaab3d9c741d57d65, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:41:07,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b873f2f74d9144109417dbd08851beff as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b873f2f74d9144109417dbd08851beff 2024-11-19T04:41:07,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b873f2f74d9144109417dbd08851beff, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:41:07,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2579823445e39a7eaea2aba4ffb4cf67 in 375ms, sequenceid=15, compaction requested=false 2024-11-19T04:41:07,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:07,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T04:41:07,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:07,308 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T04:41:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:07,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:07,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b09468f7c69c48f199464a77390f27a9_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991267027/Put/seqid=0 2024-11-19T04:41:07,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741958_1134 (size=12154) 2024-11-19T04:41:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:07,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991327404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991327404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991327437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991327438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991327439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T04:41:07,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991327558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991327558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991327558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991327560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991327560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:07,761 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b09468f7c69c48f199464a77390f27a9_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b09468f7c69c48f199464a77390f27a9_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2fdb3a90005d45f7a3d535a9c8278f26, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2fdb3a90005d45f7a3d535a9c8278f26 is 175, key is test_row_0/A:col10/1731991267027/Put/seqid=0 2024-11-19T04:41:07,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991327768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991327768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991327769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991327769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991327778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:07,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741959_1135 (size=30955) 2024-11-19T04:41:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T04:41:08,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991328078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991328078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991328078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991328078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991328089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,190 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2fdb3a90005d45f7a3d535a9c8278f26 2024-11-19T04:41:08,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/aaaff56b79cb435bac9e6966ac5c48da is 50, key is test_row_0/B:col10/1731991267027/Put/seqid=0 2024-11-19T04:41:08,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741960_1136 (size=12001) 2024-11-19T04:41:08,252 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/aaaff56b79cb435bac9e6966ac5c48da 2024-11-19T04:41:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/46cc1adb1c09457ea51b2fdc78936cc8 is 50, key is test_row_0/C:col10/1731991267027/Put/seqid=0 2024-11-19T04:41:08,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741961_1137 (size=12001) 2024-11-19T04:41:08,280 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/46cc1adb1c09457ea51b2fdc78936cc8 2024-11-19T04:41:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2fdb3a90005d45f7a3d535a9c8278f26 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26 2024-11-19T04:41:08,300 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26, entries=150, sequenceid=51, filesize=30.2 K 2024-11-19T04:41:08,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/aaaff56b79cb435bac9e6966ac5c48da as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/aaaff56b79cb435bac9e6966ac5c48da 2024-11-19T04:41:08,306 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/aaaff56b79cb435bac9e6966ac5c48da, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T04:41:08,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/46cc1adb1c09457ea51b2fdc78936cc8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/46cc1adb1c09457ea51b2fdc78936cc8 2024-11-19T04:41:08,322 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/46cc1adb1c09457ea51b2fdc78936cc8, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T04:41:08,324 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 2579823445e39a7eaea2aba4ffb4cf67 in 1016ms, sequenceid=51, compaction requested=false 2024-11-19T04:41:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:08,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T04:41:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T04:41:08,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T04:41:08,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-11-19T04:41:08,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5000 sec 2024-11-19T04:41:08,458 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T04:41:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:08,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:41:08,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:08,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a9924dc362984ece998368efa4afd111_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:08,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991328666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991328669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991328671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991328679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741962_1138 (size=12154) 2024-11-19T04:41:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991328689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991328797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991328797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991328798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991328801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991328807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T04:41:08,988 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:08,989 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T04:41:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T04:41:08,992 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:08,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:08,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:09,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991329007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991329007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991329007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991329008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991329017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T04:41:09,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:09,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a9924dc362984ece998368efa4afd111_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a9924dc362984ece998368efa4afd111_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:09,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/5b8ca380e58040f6b426f1e59c5b1491, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:09,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/5b8ca380e58040f6b426f1e59c5b1491 is 175, key is test_row_0/A:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:09,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741963_1139 (size=30955) 2024-11-19T04:41:09,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T04:41:09,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:09,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:09,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:09,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/5b8ca380e58040f6b426f1e59c5b1491 2024-11-19T04:41:09,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8f74d0da38734271956ddab48f66ab4c is 50, key is test_row_0/B:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741964_1140 (size=12001) 2024-11-19T04:41:09,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8f74d0da38734271956ddab48f66ab4c 2024-11-19T04:41:09,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/da8c8b3b43ee416093d1f7305be804ab is 50, key is test_row_0/C:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:09,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741965_1141 (size=12001) 2024-11-19T04:41:09,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/da8c8b3b43ee416093d1f7305be804ab 2024-11-19T04:41:09,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/5b8ca380e58040f6b426f1e59c5b1491 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491 2024-11-19T04:41:09,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491, entries=150, sequenceid=64, filesize=30.2 K 2024-11-19T04:41:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8f74d0da38734271956ddab48f66ab4c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8f74d0da38734271956ddab48f66ab4c 2024-11-19T04:41:09,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8f74d0da38734271956ddab48f66ab4c, entries=150, sequenceid=64, filesize=11.7 K 2024-11-19T04:41:09,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/da8c8b3b43ee416093d1f7305be804ab as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/da8c8b3b43ee416093d1f7305be804ab 2024-11-19T04:41:09,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T04:41:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991329328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991329328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991329329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991329329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T04:41:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991329329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:09,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/da8c8b3b43ee416093d1f7305be804ab, entries=150, sequenceid=64, filesize=11.7 K 2024-11-19T04:41:09,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2579823445e39a7eaea2aba4ffb4cf67 in 737ms, sequenceid=64, compaction requested=true 2024-11-19T04:41:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:09,335 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:09,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T04:41:09,335 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:09,337 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:09,337 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:09,337 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:09,337 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:09,337 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,337 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,337 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=107.5 K 2024-11-19T04:41:09,337 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1cdd5a5d2d084fbeaab3d9c741d57d65, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/aaaff56b79cb435bac9e6966ac5c48da, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8f74d0da38734271956ddab48f66ab4c] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=35.2 K 2024-11-19T04:41:09,338 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,338 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491] 2024-11-19T04:41:09,338 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdd5a5d2d084fbeaab3d9c741d57d65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991266858 2024-11-19T04:41:09,338 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95f32bb007b4ef49ddf9f42844a2a11, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991266858 2024-11-19T04:41:09,339 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fdb3a90005d45f7a3d535a9c8278f26, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731991267024 2024-11-19T04:41:09,339 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8ca380e58040f6b426f1e59c5b1491, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731991267390 2024-11-19T04:41:09,340 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting aaaff56b79cb435bac9e6966ac5c48da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731991267024 2024-11-19T04:41:09,342 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f74d0da38734271956ddab48f66ab4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731991267390 2024-11-19T04:41:09,353 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:09,355 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:09,356 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/b74e4b8dac9d416a9050624eac529fb9 is 50, key is test_row_0/B:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:09,360 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119971cf3393ce14d3caf909d7b6f3dd332_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:09,368 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119971cf3393ce14d3caf909d7b6f3dd332_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:09,368 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119971cf3393ce14d3caf909d7b6f3dd332_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:09,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741966_1142 (size=12104) 2024-11-19T04:41:09,377 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/b74e4b8dac9d416a9050624eac529fb9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b74e4b8dac9d416a9050624eac529fb9 2024-11-19T04:41:09,383 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into b74e4b8dac9d416a9050624eac529fb9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:09,383 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:09,383 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991269335; duration=0sec 2024-11-19T04:41:09,383 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:09,383 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:09,383 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:09,385 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:09,386 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:09,386 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,386 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b873f2f74d9144109417dbd08851beff, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/46cc1adb1c09457ea51b2fdc78936cc8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/da8c8b3b43ee416093d1f7305be804ab] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=35.2 K 2024-11-19T04:41:09,386 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b873f2f74d9144109417dbd08851beff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991266858 2024-11-19T04:41:09,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741967_1143 (size=4469) 2024-11-19T04:41:09,388 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 46cc1adb1c09457ea51b2fdc78936cc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731991267024 2024-11-19T04:41:09,389 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting da8c8b3b43ee416093d1f7305be804ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731991267390 2024-11-19T04:41:09,390 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#124 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:09,391 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/1512e9fcf07947918ab72095eb0d15e0 is 175, key is test_row_0/A:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:09,408 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:09,409 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/1b60efce4ccf402abfbed20e53b38339 is 50, key is test_row_0/C:col10/1731991268595/Put/seqid=0 2024-11-19T04:41:09,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741968_1144 (size=31058) 2024-11-19T04:41:09,429 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/1512e9fcf07947918ab72095eb0d15e0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1512e9fcf07947918ab72095eb0d15e0 2024-11-19T04:41:09,437 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into 1512e9fcf07947918ab72095eb0d15e0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:09,437 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:09,437 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991269335; duration=0sec 2024-11-19T04:41:09,437 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:09,438 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741969_1145 (size=12104) 2024-11-19T04:41:09,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T04:41:09,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:09,487 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:41:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:09,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ca875193c4b84dd8a5521ec933c5ca86_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991268665/Put/seqid=0 2024-11-19T04:41:09,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741970_1146 (size=12154) 2024-11-19T04:41:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T04:41:09,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:09,851 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/1b60efce4ccf402abfbed20e53b38339 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/1b60efce4ccf402abfbed20e53b38339 2024-11-19T04:41:09,858 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 1b60efce4ccf402abfbed20e53b38339(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:09,858 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:09,858 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991269335; duration=0sec 2024-11-19T04:41:09,858 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:09,858 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:09,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991329850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991329854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991329854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991329857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991329859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:09,929 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ca875193c4b84dd8a5521ec933c5ca86_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ca875193c4b84dd8a5521ec933c5ca86_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:09,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/dec654b0f00945c994d59508b2909ec0, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:09,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/dec654b0f00945c994d59508b2909ec0 is 175, key is test_row_0/A:col10/1731991268665/Put/seqid=0 2024-11-19T04:41:09,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741971_1147 (size=30955) 2024-11-19T04:41:09,951 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/dec654b0f00945c994d59508b2909ec0 2024-11-19T04:41:09,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991329966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991329967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/997b54ae03144ee4a799e2e2e1096832 is 50, key is test_row_0/B:col10/1731991268665/Put/seqid=0 2024-11-19T04:41:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991329977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991329977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991329977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:09,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741972_1148 (size=12001) 2024-11-19T04:41:09,989 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/997b54ae03144ee4a799e2e2e1096832 2024-11-19T04:41:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/0f7a67d29d664699931396a53e173686 is 50, key is test_row_0/C:col10/1731991268665/Put/seqid=0 2024-11-19T04:41:10,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741973_1149 (size=12001) 2024-11-19T04:41:10,024 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/0f7a67d29d664699931396a53e173686 2024-11-19T04:41:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/dec654b0f00945c994d59508b2909ec0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0 2024-11-19T04:41:10,037 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0, entries=150, sequenceid=86, filesize=30.2 K 2024-11-19T04:41:10,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/997b54ae03144ee4a799e2e2e1096832 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/997b54ae03144ee4a799e2e2e1096832 2024-11-19T04:41:10,062 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/997b54ae03144ee4a799e2e2e1096832, entries=150, sequenceid=86, filesize=11.7 K 2024-11-19T04:41:10,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/0f7a67d29d664699931396a53e173686 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/0f7a67d29d664699931396a53e173686 2024-11-19T04:41:10,074 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/0f7a67d29d664699931396a53e173686, entries=150, sequenceid=86, filesize=11.7 K 2024-11-19T04:41:10,076 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2579823445e39a7eaea2aba4ffb4cf67 in 588ms, sequenceid=86, compaction requested=false 2024-11-19T04:41:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T04:41:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T04:41:10,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T04:41:10,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-11-19T04:41:10,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0940 sec 2024-11-19T04:41:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T04:41:10,148 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:10,149 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T04:41:10,152 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:10,153 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:10,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T04:41:10,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:41:10,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:10,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:10,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:10,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:10,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:10,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:10,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b44fa8c28d724321b28a016168d27999_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741974_1150 (size=12154) 2024-11-19T04:41:10,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991330216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,223 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b44fa8c28d724321b28a016168d27999_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b44fa8c28d724321b28a016168d27999_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:10,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991330219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/274d5f8070674986a4cc17f941a652d9, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:10,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/274d5f8070674986a4cc17f941a652d9 is 175, key is test_row_0/A:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991330221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991330222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991330223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741975_1151 (size=30955) 2024-11-19T04:41:10,239 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/274d5f8070674986a4cc17f941a652d9 2024-11-19T04:41:10,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/7ec4532f4e4e48fd962d88d051438a84 is 50, key is test_row_0/B:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T04:41:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741976_1152 (size=12001) 2024-11-19T04:41:10,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/7ec4532f4e4e48fd962d88d051438a84 2024-11-19T04:41:10,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/87e1c45d266b4779a309dc67dc22e3e9 is 50, key is test_row_0/C:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741977_1153 (size=12001) 2024-11-19T04:41:10,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T04:41:10,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:10,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991330327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991330337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991330337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991330338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991330338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T04:41:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T04:41:10,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991330537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991330547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991330548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991330548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991330559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T04:41:10,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:10,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/87e1c45d266b4779a309dc67dc22e3e9 2024-11-19T04:41:10,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/274d5f8070674986a4cc17f941a652d9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9 2024-11-19T04:41:10,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9, entries=150, sequenceid=101, filesize=30.2 K 2024-11-19T04:41:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/7ec4532f4e4e48fd962d88d051438a84 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/7ec4532f4e4e48fd962d88d051438a84 2024-11-19T04:41:10,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/7ec4532f4e4e48fd962d88d051438a84, entries=150, sequenceid=101, filesize=11.7 K 2024-11-19T04:41:10,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/87e1c45d266b4779a309dc67dc22e3e9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/87e1c45d266b4779a309dc67dc22e3e9 2024-11-19T04:41:10,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/87e1c45d266b4779a309dc67dc22e3e9, entries=150, sequenceid=101, filesize=11.7 K 2024-11-19T04:41:10,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 2579823445e39a7eaea2aba4ffb4cf67 in 550ms, sequenceid=101, compaction requested=true 2024-11-19T04:41:10,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:10,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:10,731 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:10,731 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:10,734 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:10,734 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:10,734 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,734 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1512e9fcf07947918ab72095eb0d15e0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=90.8 K 2024-11-19T04:41:10,734 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,734 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1512e9fcf07947918ab72095eb0d15e0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9] 2024-11-19T04:41:10,734 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:10,735 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:10,735 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,735 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1512e9fcf07947918ab72095eb0d15e0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731991267390 2024-11-19T04:41:10,735 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b74e4b8dac9d416a9050624eac529fb9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/997b54ae03144ee4a799e2e2e1096832, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/7ec4532f4e4e48fd962d88d051438a84] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=35.3 K 2024-11-19T04:41:10,737 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dec654b0f00945c994d59508b2909ec0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731991268654 2024-11-19T04:41:10,737 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 274d5f8070674986a4cc17f941a652d9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731991269856 2024-11-19T04:41:10,738 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b74e4b8dac9d416a9050624eac529fb9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731991267390 2024-11-19T04:41:10,739 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 997b54ae03144ee4a799e2e2e1096832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731991268654 2024-11-19T04:41:10,739 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec4532f4e4e48fd962d88d051438a84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731991269856 2024-11-19T04:41:10,754 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:10,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,768 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:10,771 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:10,771 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/16c0fea3c21a45d58cbccf32202c89b4 is 50, key is test_row_0/B:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,781 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111959e56ae591834e9bb9e11293270d9b99_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:10,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111959e56ae591834e9bb9e11293270d9b99_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:10,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111959e56ae591834e9bb9e11293270d9b99_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T04:41:10,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741978_1154 (size=12207) 2024-11-19T04:41:10,840 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/16c0fea3c21a45d58cbccf32202c89b4 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/16c0fea3c21a45d58cbccf32202c89b4 2024-11-19T04:41:10,849 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 16c0fea3c21a45d58cbccf32202c89b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:10,849 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:10,849 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991270731; duration=0sec 2024-11-19T04:41:10,850 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:10,850 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:10,850 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:10,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e8854b0de3064a4f9f51460f028d5d4f_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991270218/Put/seqid=0 2024-11-19T04:41:10,853 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:10,853 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:10,853 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:10,854 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/1b60efce4ccf402abfbed20e53b38339, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/0f7a67d29d664699931396a53e173686, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/87e1c45d266b4779a309dc67dc22e3e9] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=35.3 K 2024-11-19T04:41:10,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:10,856 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b60efce4ccf402abfbed20e53b38339, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731991267390 2024-11-19T04:41:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:10,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7a67d29d664699931396a53e173686, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731991268654 2024-11-19T04:41:10,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e1c45d266b4779a309dc67dc22e3e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731991269856 2024-11-19T04:41:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741979_1155 (size=4469) 2024-11-19T04:41:10,876 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#132 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:10,877 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b87db6beeb0344078d468e164887c94f is 175, key is test_row_0/A:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,887 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:10,887 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/713230dd3d1647dcb050fbc47c5d5d50 is 50, key is test_row_0/C:col10/1731991269856/Put/seqid=0 2024-11-19T04:41:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991330882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991330884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991330889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991330891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991330893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:10,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741980_1156 (size=12154) 2024-11-19T04:41:10,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:10,917 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e8854b0de3064a4f9f51460f028d5d4f_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e8854b0de3064a4f9f51460f028d5d4f_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:10,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741981_1157 (size=31161) 2024-11-19T04:41:10,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/7a7ac4ba39d74b798221de29b11e3e36, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:10,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/7a7ac4ba39d74b798221de29b11e3e36 is 175, key is test_row_0/A:col10/1731991270218/Put/seqid=0 2024-11-19T04:41:10,933 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b87db6beeb0344078d468e164887c94f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b87db6beeb0344078d468e164887c94f 2024-11-19T04:41:10,941 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into b87db6beeb0344078d468e164887c94f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:10,941 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:10,941 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991270730; duration=0sec 2024-11-19T04:41:10,941 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:10,942 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:10,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741983_1159 (size=30955) 2024-11-19T04:41:10,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741982_1158 (size=12207) 2024-11-19T04:41:10,970 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/7a7ac4ba39d74b798221de29b11e3e36 2024-11-19T04:41:10,976 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/713230dd3d1647dcb050fbc47c5d5d50 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/713230dd3d1647dcb050fbc47c5d5d50 2024-11-19T04:41:10,987 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 713230dd3d1647dcb050fbc47c5d5d50(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:10,987 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:10,987 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991270731; duration=0sec 2024-11-19T04:41:10,987 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:10,987 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:10,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/338afe58dc03481c94ac92082fb4e767 is 50, key is test_row_0/B:col10/1731991270218/Put/seqid=0 2024-11-19T04:41:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991330997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991330997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991330997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741984_1160 (size=12001) 2024-11-19T04:41:11,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991331008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991331008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991331207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991331207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991331207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991331217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991331217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T04:41:11,411 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/338afe58dc03481c94ac92082fb4e767 2024-11-19T04:41:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/44b2bc99d46a4b97bbd76b0b00d6ef6a is 50, key is test_row_0/C:col10/1731991270218/Put/seqid=0 2024-11-19T04:41:11,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741985_1161 (size=12001) 2024-11-19T04:41:11,436 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/44b2bc99d46a4b97bbd76b0b00d6ef6a 2024-11-19T04:41:11,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/7a7ac4ba39d74b798221de29b11e3e36 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36 2024-11-19T04:41:11,448 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36, entries=150, sequenceid=120, filesize=30.2 K 2024-11-19T04:41:11,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/338afe58dc03481c94ac92082fb4e767 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/338afe58dc03481c94ac92082fb4e767 2024-11-19T04:41:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,457 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/338afe58dc03481c94ac92082fb4e767, entries=150, sequenceid=120, filesize=11.7 K 2024-11-19T04:41:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/44b2bc99d46a4b97bbd76b0b00d6ef6a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/44b2bc99d46a4b97bbd76b0b00d6ef6a 2024-11-19T04:41:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,477 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/44b2bc99d46a4b97bbd76b0b00d6ef6a, entries=150, sequenceid=120, filesize=11.7 K 2024-11-19T04:41:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,478 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2579823445e39a7eaea2aba4ffb4cf67 in 710ms, sequenceid=120, compaction requested=false 2024-11-19T04:41:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:11,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T04:41:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T04:41:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T04:41:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-19T04:41:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3350 sec 2024-11-19T04:41:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:11,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:41:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:11,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190f24643bc4c74ce1830c06fe967d5837_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741986_1162 (size=12304) 2024-11-19T04:41:11,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:11,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991331565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,575 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190f24643bc4c74ce1830c06fe967d5837_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190f24643bc4c74ce1830c06fe967d5837_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,576 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/95fb4d7b9ba24c44b1abe3f999e4c891, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:11,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991331568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/95fb4d7b9ba24c44b1abe3f999e4c891 is 175, key is test_row_0/A:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991331570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991331572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991331572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741987_1163 (size=31101) 2024-11-19T04:41:11,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/95fb4d7b9ba24c44b1abe3f999e4c891 2024-11-19T04:41:11,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1625a28354e849f794c7d3e151c67c45 is 50, key is test_row_0/B:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741988_1164 (size=9757) 2024-11-19T04:41:11,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1625a28354e849f794c7d3e151c67c45 2024-11-19T04:41:11,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f39ef1ca120b424d8fbeef2c4d7f2941 is 50, key is test_row_0/C:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741989_1165 (size=9757) 2024-11-19T04:41:11,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f39ef1ca120b424d8fbeef2c4d7f2941 2024-11-19T04:41:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991331688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991331688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991331688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991331688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/95fb4d7b9ba24c44b1abe3f999e4c891 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991331688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891, entries=150, sequenceid=137, filesize=30.4 K 2024-11-19T04:41:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1625a28354e849f794c7d3e151c67c45 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1625a28354e849f794c7d3e151c67c45 2024-11-19T04:41:11,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1625a28354e849f794c7d3e151c67c45, entries=100, sequenceid=137, filesize=9.5 K 2024-11-19T04:41:11,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f39ef1ca120b424d8fbeef2c4d7f2941 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f39ef1ca120b424d8fbeef2c4d7f2941 2024-11-19T04:41:11,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f39ef1ca120b424d8fbeef2c4d7f2941, entries=100, sequenceid=137, filesize=9.5 K 2024-11-19T04:41:11,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 2579823445e39a7eaea2aba4ffb4cf67 in 188ms, sequenceid=137, compaction requested=true 2024-11-19T04:41:11,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:11,728 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:11,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:11,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:11,729 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:11,729 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:11,729 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:11,730 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:11,730 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b87db6beeb0344078d468e164887c94f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=91.0 K 2024-11-19T04:41:11,730 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:11,730 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b87db6beeb0344078d468e164887c94f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891] 2024-11-19T04:41:11,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:11,730 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b87db6beeb0344078d468e164887c94f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731991269856 2024-11-19T04:41:11,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:11,731 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a7ac4ba39d74b798221de29b11e3e36, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731991270218 2024-11-19T04:41:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:11,731 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:11,731 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:11,731 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:11,732 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/16c0fea3c21a45d58cbccf32202c89b4, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/338afe58dc03481c94ac92082fb4e767, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1625a28354e849f794c7d3e151c67c45] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=33.2 K 2024-11-19T04:41:11,734 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c0fea3c21a45d58cbccf32202c89b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731991269856 2024-11-19T04:41:11,734 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95fb4d7b9ba24c44b1abe3f999e4c891, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991270888 2024-11-19T04:41:11,735 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 338afe58dc03481c94ac92082fb4e767, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731991270218 2024-11-19T04:41:11,736 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1625a28354e849f794c7d3e151c67c45, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991270890 2024-11-19T04:41:11,757 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:11,767 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:11,768 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/296a84ae064940e58ace92ab5acbe539 is 50, key is test_row_0/B:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,784 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b50113118f0042478660524f27719c56_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:11,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b50113118f0042478660524f27719c56_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:11,788 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b50113118f0042478660524f27719c56_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:11,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741990_1166 (size=12409) 2024-11-19T04:41:11,843 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/296a84ae064940e58ace92ab5acbe539 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/296a84ae064940e58ace92ab5acbe539 2024-11-19T04:41:11,854 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 296a84ae064940e58ace92ab5acbe539(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:11,854 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:11,854 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991271729; duration=0sec 2024-11-19T04:41:11,854 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:11,854 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:11,855 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:11,856 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:11,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:11,857 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:11,857 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/713230dd3d1647dcb050fbc47c5d5d50, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/44b2bc99d46a4b97bbd76b0b00d6ef6a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f39ef1ca120b424d8fbeef2c4d7f2941] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=33.2 K 2024-11-19T04:41:11,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 713230dd3d1647dcb050fbc47c5d5d50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731991269856 2024-11-19T04:41:11,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741991_1167 (size=4469) 2024-11-19T04:41:11,859 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b2bc99d46a4b97bbd76b0b00d6ef6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731991270218 2024-11-19T04:41:11,859 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f39ef1ca120b424d8fbeef2c4d7f2941, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991270890 2024-11-19T04:41:11,862 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#141 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:11,863 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/727ea5f9ba614da68fecad1d221dfa31 is 175, key is test_row_0/A:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741992_1168 (size=31470) 2024-11-19T04:41:11,894 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#143 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:11,894 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/85f00871e52d406dad2ff15a363e65dd is 50, key is test_row_0/C:col10/1731991270890/Put/seqid=0 2024-11-19T04:41:11,895 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/727ea5f9ba614da68fecad1d221dfa31 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/727ea5f9ba614da68fecad1d221dfa31 2024-11-19T04:41:11,905 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into 727ea5f9ba614da68fecad1d221dfa31(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:11,905 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:11,905 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991271727; duration=0sec 2024-11-19T04:41:11,905 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:11,905 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:11,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:41:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741993_1169 (size=12409) 2024-11-19T04:41:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991331938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194be80f352f3349cd907a3ec1d622525b_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991331939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991331939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991331942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991331942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:11,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741994_1170 (size=19774) 2024-11-19T04:41:12,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991332047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991332057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991332058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991332058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991332058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991332267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991332267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991332268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991332268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991332268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T04:41:12,317 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:12,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T04:41:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T04:41:12,320 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:12,321 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:12,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:12,346 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/85f00871e52d406dad2ff15a363e65dd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/85f00871e52d406dad2ff15a363e65dd 2024-11-19T04:41:12,351 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 85f00871e52d406dad2ff15a363e65dd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:12,351 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:12,351 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991271731; duration=0sec 2024-11-19T04:41:12,351 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:12,351 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:12,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,398 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194be80f352f3349cd907a3ec1d622525b_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be80f352f3349cd907a3ec1d622525b_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:12,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a3967760c68741c6b955b707261496f1, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:12,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a3967760c68741c6b955b707261496f1 is 175, key is test_row_0/A:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741995_1171 (size=57033) 2024-11-19T04:41:12,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a3967760c68741c6b955b707261496f1 2024-11-19T04:41:12,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/d248b0bc723341ada86e133e63bad2db is 50, key is test_row_0/B:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T04:41:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741996_1172 (size=12151) 2024-11-19T04:41:12,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/d248b0bc723341ada86e133e63bad2db 2024-11-19T04:41:12,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/e1601f400a734cdaa15fc21e95f82cc5 is 50, key is test_row_0/C:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:12,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741997_1173 (size=12151) 2024-11-19T04:41:12,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/e1601f400a734cdaa15fc21e95f82cc5 2024-11-19T04:41:12,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a3967760c68741c6b955b707261496f1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1 2024-11-19T04:41:12,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T04:41:12,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1, entries=300, sequenceid=161, filesize=55.7 K 2024-11-19T04:41:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/d248b0bc723341ada86e133e63bad2db as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d248b0bc723341ada86e133e63bad2db 2024-11-19T04:41:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d248b0bc723341ada86e133e63bad2db, entries=150, sequenceid=161, filesize=11.9 K 2024-11-19T04:41:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/e1601f400a734cdaa15fc21e95f82cc5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e1601f400a734cdaa15fc21e95f82cc5 2024-11-19T04:41:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e1601f400a734cdaa15fc21e95f82cc5, entries=150, sequenceid=161, filesize=11.9 K 2024-11-19T04:41:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 2579823445e39a7eaea2aba4ffb4cf67 in 594ms, sequenceid=161, compaction requested=false 2024-11-19T04:41:12,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:12,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:41:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T04:41:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:12,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T04:41:12,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191b9b161ff472425989ed17d304d2d2ce_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_1/A:col10/1731991271936/Put/seqid=0 2024-11-19T04:41:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:12,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991272655, max timestamp=1731991272670Num uniques 859; 2024-11-19T04:41:12,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:12,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991272655, max timestamp=1731991272672Num uniques 859; 2024-11-19T04:41:12,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:12,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991272655, max timestamp=1731991272670Num uniques 859; 2024-11-19T04:41:12,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:12,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741998_1174 (size=12304) 2024-11-19T04:41:12,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T04:41:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991332773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991332785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991332785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991332787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991332787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991332898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991332898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991332898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:12,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T04:41:12,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:12,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:12,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T04:41:13,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T04:41:13,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:13,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:13,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:13,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991333109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991333109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991333109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991333110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991333110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,119 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:13,124 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191b9b161ff472425989ed17d304d2d2ce_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191b9b161ff472425989ed17d304d2d2ce_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:13,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/1f462375f25644309c93d005a840db9a, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:13,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/1f462375f25644309c93d005a840db9a is 175, key is test_row_1/A:col10/1731991271936/Put/seqid=0 2024-11-19T04:41:13,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741999_1175 (size=31101) 2024-11-19T04:41:13,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/1f462375f25644309c93d005a840db9a 2024-11-19T04:41:13,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/c5b33853e4f44571815994332366ed1e is 50, key is test_row_1/B:col10/1731991271936/Put/seqid=0 2024-11-19T04:41:13,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742000_1176 (size=9757) 2024-11-19T04:41:13,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/c5b33853e4f44571815994332366ed1e 2024-11-19T04:41:13,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/bd48ffca86bc44bda6b4ceef68851c81 is 50, key is test_row_1/C:col10/1731991271936/Put/seqid=0 2024-11-19T04:41:13,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742001_1177 (size=9757) 2024-11-19T04:41:13,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/bd48ffca86bc44bda6b4ceef68851c81 2024-11-19T04:41:13,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/1f462375f25644309c93d005a840db9a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a 2024-11-19T04:41:13,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a, entries=150, sequenceid=173, filesize=30.4 K 2024-11-19T04:41:13,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/c5b33853e4f44571815994332366ed1e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/c5b33853e4f44571815994332366ed1e 2024-11-19T04:41:13,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/c5b33853e4f44571815994332366ed1e, entries=100, sequenceid=173, filesize=9.5 K 2024-11-19T04:41:13,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/bd48ffca86bc44bda6b4ceef68851c81 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/bd48ffca86bc44bda6b4ceef68851c81 2024-11-19T04:41:13,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/bd48ffca86bc44bda6b4ceef68851c81, entries=100, sequenceid=173, filesize=9.5 K 2024-11-19T04:41:13,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2579823445e39a7eaea2aba4ffb4cf67 in 605ms, sequenceid=173, compaction requested=true 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:13,225 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:13,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:13,225 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:13,228 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:13,228 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:13,228 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,229 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/296a84ae064940e58ace92ab5acbe539, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d248b0bc723341ada86e133e63bad2db, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/c5b33853e4f44571815994332366ed1e] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=33.5 K 2024-11-19T04:41:13,229 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119604 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:13,229 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:13,229 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,229 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/727ea5f9ba614da68fecad1d221dfa31, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=116.8 K 2024-11-19T04:41:13,229 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,229 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/727ea5f9ba614da68fecad1d221dfa31, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a] 2024-11-19T04:41:13,230 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 727ea5f9ba614da68fecad1d221dfa31, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991270221 2024-11-19T04:41:13,230 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 296a84ae064940e58ace92ab5acbe539, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991270221 2024-11-19T04:41:13,230 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d248b0bc723341ada86e133e63bad2db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731991271913 2024-11-19T04:41:13,230 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3967760c68741c6b955b707261496f1, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731991271557 2024-11-19T04:41:13,230 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f462375f25644309c93d005a840db9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731991271936 2024-11-19T04:41:13,231 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b33853e4f44571815994332366ed1e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731991271936 2024-11-19T04:41:13,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T04:41:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,261 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-19T04:41:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:13,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:13,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:13,267 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:13,267 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f7a2cb1b9cf2432486ea1341061d73d0 is 50, key is test_row_0/B:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:13,270 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:13,285 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199e17d39b7ebe4e6391dd2b2e14f6f552_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:13,287 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199e17d39b7ebe4e6391dd2b2e14f6f552_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:13,287 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e17d39b7ebe4e6391dd2b2e14f6f552_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111933d687cb618e4bd3af4b660f558ab439_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991272783/Put/seqid=0 2024-11-19T04:41:13,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742002_1178 (size=12561) 2024-11-19T04:41:13,324 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f7a2cb1b9cf2432486ea1341061d73d0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f7a2cb1b9cf2432486ea1341061d73d0 2024-11-19T04:41:13,334 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into f7a2cb1b9cf2432486ea1341061d73d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:13,335 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:13,335 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991273225; duration=0sec 2024-11-19T04:41:13,335 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:13,335 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:13,335 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:13,339 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:13,342 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:13,342 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,343 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/85f00871e52d406dad2ff15a363e65dd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e1601f400a734cdaa15fc21e95f82cc5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/bd48ffca86bc44bda6b4ceef68851c81] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=33.5 K 2024-11-19T04:41:13,343 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f00871e52d406dad2ff15a363e65dd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991270221 2024-11-19T04:41:13,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742004_1180 (size=4469) 2024-11-19T04:41:13,344 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e1601f400a734cdaa15fc21e95f82cc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731991271913 2024-11-19T04:41:13,346 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting bd48ffca86bc44bda6b4ceef68851c81, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731991271936 2024-11-19T04:41:13,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742003_1179 (size=12304) 2024-11-19T04:41:13,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:13,357 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111933d687cb618e4bd3af4b660f558ab439_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111933d687cb618e4bd3af4b660f558ab439_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2984846e9fda4c8fbd348f2fdacca210, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:13,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2984846e9fda4c8fbd348f2fdacca210 is 175, key is test_row_0/A:col10/1731991272783/Put/seqid=0 2024-11-19T04:41:13,373 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#153 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:13,374 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/5838c4623088426ba6d906ea708602ad is 50, key is test_row_0/C:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:13,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742005_1181 (size=31105) 2024-11-19T04:41:13,382 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2984846e9fda4c8fbd348f2fdacca210 2024-11-19T04:41:13,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742006_1182 (size=12561) 2024-11-19T04:41:13,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/0fa3a82edb024e07a925e4399bf8f986 is 50, key is test_row_0/B:col10/1731991272783/Put/seqid=0 2024-11-19T04:41:13,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742007_1183 (size=12151) 2024-11-19T04:41:13,405 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/0fa3a82edb024e07a925e4399bf8f986 2024-11-19T04:41:13,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/ec758aef7b1a42da8352ab3402140ae1 is 50, key is test_row_0/C:col10/1731991272783/Put/seqid=0 2024-11-19T04:41:13,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:13,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742008_1184 (size=12151) 2024-11-19T04:41:13,443 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/ec758aef7b1a42da8352ab3402140ae1 2024-11-19T04:41:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991333432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991333434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991333442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991333444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991333446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/2984846e9fda4c8fbd348f2fdacca210 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210 2024-11-19T04:41:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T04:41:13,458 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210, entries=150, sequenceid=207, filesize=30.4 K 2024-11-19T04:41:13,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/0fa3a82edb024e07a925e4399bf8f986 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/0fa3a82edb024e07a925e4399bf8f986 2024-11-19T04:41:13,467 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/0fa3a82edb024e07a925e4399bf8f986, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T04:41:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/ec758aef7b1a42da8352ab3402140ae1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ec758aef7b1a42da8352ab3402140ae1 2024-11-19T04:41:13,475 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ec758aef7b1a42da8352ab3402140ae1, entries=150, sequenceid=207, filesize=11.9 K 2024-11-19T04:41:13,476 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 2579823445e39a7eaea2aba4ffb4cf67 in 215ms, sequenceid=207, compaction requested=false 2024-11-19T04:41:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T04:41:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T04:41:13,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T04:41:13,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-11-19T04:41:13,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1650 sec 2024-11-19T04:41:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:13,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:41:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:13,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ad984f20646545378a543487fbcabb51_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742009_1185 (size=14794) 2024-11-19T04:41:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991333620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991333621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991333622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991333623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991333624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991333737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991333737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991333737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991333737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991333737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,746 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#151 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:13,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/bb34b28688d54289ba57b60af799dc8e is 175, key is test_row_0/A:col10/1731991271913/Put/seqid=0 2024-11-19T04:41:13,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742010_1186 (size=31622) 2024-11-19T04:41:13,778 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/bb34b28688d54289ba57b60af799dc8e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/bb34b28688d54289ba57b60af799dc8e 2024-11-19T04:41:13,786 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into bb34b28688d54289ba57b60af799dc8e(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:13,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:13,787 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991273225; duration=0sec 2024-11-19T04:41:13,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:13,787 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:13,804 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/5838c4623088426ba6d906ea708602ad as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5838c4623088426ba6d906ea708602ad 2024-11-19T04:41:13,813 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 5838c4623088426ba6d906ea708602ad(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:13,813 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:13,813 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991273225; duration=0sec 2024-11-19T04:41:13,813 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:13,813 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:13,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991333957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:13,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:13,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:13,996 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ad984f20646545378a543487fbcabb51_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad984f20646545378a543487fbcabb51_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:14,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/fbbdac5d85d6448389f22425d65173d6, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:14,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/fbbdac5d85d6448389f22425d65173d6 is 175, key is test_row_0/A:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:14,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742011_1187 (size=39749) 2024-11-19T04:41:14,016 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/fbbdac5d85d6448389f22425d65173d6 2024-11-19T04:41:14,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/bc817a591d7641f3826b2b7434d905b3 is 50, key is test_row_0/B:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:14,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742012_1188 (size=12151) 2024-11-19T04:41:14,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/bc817a591d7641f3826b2b7434d905b3 2024-11-19T04:41:14,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/47a98436c6b94415b516d7e641543649 is 50, key is test_row_0/C:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:14,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742013_1189 (size=12151) 2024-11-19T04:41:14,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991334258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991334258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991334268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991334268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991334268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T04:41:14,469 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:14,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T04:41:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T04:41:14,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:14,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:14,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:14,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/47a98436c6b94415b516d7e641543649 2024-11-19T04:41:14,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/fbbdac5d85d6448389f22425d65173d6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6 2024-11-19T04:41:14,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6, entries=200, sequenceid=220, filesize=38.8 K 2024-11-19T04:41:14,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/bc817a591d7641f3826b2b7434d905b3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/bc817a591d7641f3826b2b7434d905b3 2024-11-19T04:41:14,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/bc817a591d7641f3826b2b7434d905b3, entries=150, sequenceid=220, filesize=11.9 K 2024-11-19T04:41:14,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/47a98436c6b94415b516d7e641543649 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/47a98436c6b94415b516d7e641543649 2024-11-19T04:41:14,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/47a98436c6b94415b516d7e641543649, entries=150, sequenceid=220, filesize=11.9 K 2024-11-19T04:41:14,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2579823445e39a7eaea2aba4ffb4cf67 in 1004ms, sequenceid=220, compaction requested=true 2024-11-19T04:41:14,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:14,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:14,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:14,566 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:14,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:14,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:14,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:14,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:14,567 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:14,567 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102476 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:14,568 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:14,568 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:14,568 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/bb34b28688d54289ba57b60af799dc8e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=100.1 K 2024-11-19T04:41:14,568 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:14,568 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/bb34b28688d54289ba57b60af799dc8e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6] 2024-11-19T04:41:14,568 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:14,568 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:14,568 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:14,568 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f7a2cb1b9cf2432486ea1341061d73d0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/0fa3a82edb024e07a925e4399bf8f986, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/bc817a591d7641f3826b2b7434d905b3] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.0 K 2024-11-19T04:41:14,568 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb34b28688d54289ba57b60af799dc8e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731991271913 2024-11-19T04:41:14,569 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a2cb1b9cf2432486ea1341061d73d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731991271913 2024-11-19T04:41:14,569 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2984846e9fda4c8fbd348f2fdacca210, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731991272783 2024-11-19T04:41:14,569 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fa3a82edb024e07a925e4399bf8f986, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731991272783 2024-11-19T04:41:14,569 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbbdac5d85d6448389f22425d65173d6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731991273425 2024-11-19T04:41:14,570 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting bc817a591d7641f3826b2b7434d905b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731991273425 2024-11-19T04:41:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T04:41:14,586 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:14,595 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:14,596 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/995870c67a7a445b8944e4acf45b5f72 is 50, key is test_row_0/B:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:14,599 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119aa3f49ded4434648a0d470099636e47e_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:14,601 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119aa3f49ded4434648a0d470099636e47e_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:14,601 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aa3f49ded4434648a0d470099636e47e_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:14,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:14,630 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:14,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742014_1190 (size=12663) 2024-11-19T04:41:14,652 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/995870c67a7a445b8944e4acf45b5f72 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/995870c67a7a445b8944e4acf45b5f72 2024-11-19T04:41:14,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742015_1191 (size=4469) 2024-11-19T04:41:14,661 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#159 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:14,662 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/0166d2f5b48f486da4fcc7917b49600a is 175, key is test_row_0/A:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:14,669 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 995870c67a7a445b8944e4acf45b5f72(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:14,670 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:14,670 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991274566; duration=0sec 2024-11-19T04:41:14,670 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:14,670 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:14,670 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:14,672 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:14,672 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:14,672 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:14,672 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5838c4623088426ba6d906ea708602ad, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ec758aef7b1a42da8352ab3402140ae1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/47a98436c6b94415b516d7e641543649] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.0 K 2024-11-19T04:41:14,672 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5838c4623088426ba6d906ea708602ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731991271913 2024-11-19T04:41:14,673 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ec758aef7b1a42da8352ab3402140ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731991272783 2024-11-19T04:41:14,674 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a98436c6b94415b516d7e641543649, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731991273425 2024-11-19T04:41:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ff14c2dc0744a74bc71bceef1915469_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991273618/Put/seqid=0 2024-11-19T04:41:14,693 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#162 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:14,693 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/753bafeb219f43b39ebc041eca5c5233 is 50, key is test_row_0/C:col10/1731991273439/Put/seqid=0 2024-11-19T04:41:14,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742016_1192 (size=31617) 2024-11-19T04:41:14,716 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/0166d2f5b48f486da4fcc7917b49600a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/0166d2f5b48f486da4fcc7917b49600a 2024-11-19T04:41:14,729 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into 0166d2f5b48f486da4fcc7917b49600a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:14,729 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:14,729 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991274566; duration=0sec 2024-11-19T04:41:14,729 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:14,729 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742017_1193 (size=12304) 2024-11-19T04:41:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:14,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742018_1194 (size=12663) 2024-11-19T04:41:14,752 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411190ff14c2dc0744a74bc71bceef1915469_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ff14c2dc0744a74bc71bceef1915469_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d2ce5598e97649d4a7b910453147761f, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d2ce5598e97649d4a7b910453147761f is 175, key is test_row_0/A:col10/1731991273618/Put/seqid=0 2024-11-19T04:41:14,762 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/753bafeb219f43b39ebc041eca5c5233 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/753bafeb219f43b39ebc041eca5c5233 2024-11-19T04:41:14,770 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 753bafeb219f43b39ebc041eca5c5233(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:14,770 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:14,770 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991274566; duration=0sec 2024-11-19T04:41:14,770 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:14,770 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:14,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T04:41:14,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991334787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991334799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991334801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742019_1195 (size=31105) 2024-11-19T04:41:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,820 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d2ce5598e97649d4a7b910453147761f 2024-11-19T04:41:14,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991334809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991334809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/07b01763e336452f9f4369b0b1cfe242 is 50, key is test_row_0/B:col10/1731991273618/Put/seqid=0 2024-11-19T04:41:14,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742020_1196 (size=12151) 2024-11-19T04:41:14,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991334917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991334917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991334927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991334928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T04:41:15,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991335127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991335127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991335147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991335147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,268 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/07b01763e336452f9f4369b0b1cfe242 2024-11-19T04:41:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/4681891d78f5467a924f3576de159e58 is 50, key is test_row_0/C:col10/1731991273618/Put/seqid=0 2024-11-19T04:41:15,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742021_1197 (size=12151) 2024-11-19T04:41:15,292 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/4681891d78f5467a924f3576de159e58 2024-11-19T04:41:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d2ce5598e97649d4a7b910453147761f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f 2024-11-19T04:41:15,306 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f, entries=150, sequenceid=242, filesize=30.4 K 2024-11-19T04:41:15,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/07b01763e336452f9f4369b0b1cfe242 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/07b01763e336452f9f4369b0b1cfe242 2024-11-19T04:41:15,333 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/07b01763e336452f9f4369b0b1cfe242, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T04:41:15,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/4681891d78f5467a924f3576de159e58 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4681891d78f5467a924f3576de159e58 2024-11-19T04:41:15,352 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4681891d78f5467a924f3576de159e58, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T04:41:15,353 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2579823445e39a7eaea2aba4ffb4cf67 in 723ms, sequenceid=242, compaction requested=false 2024-11-19T04:41:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T04:41:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T04:41:15,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T04:41:15,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 881 msec 2024-11-19T04:41:15,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 891 msec 2024-11-19T04:41:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:15,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:41:15,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:15,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:15,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:15,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:15,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:15,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:15,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119477a94640fc9465e80fd23f3cd0cc5d1_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742022_1198 (size=17384) 2024-11-19T04:41:15,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991335493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991335495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991335495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991335498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T04:41:15,607 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:15,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991335607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T04:41:15,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991335607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991335607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,611 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991335607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,613 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:15,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T04:41:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T04:41:15,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T04:41:15,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:15,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:15,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991335817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,822 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:15,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:15,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:15,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119477a94640fc9465e80fd23f3cd0cc5d1_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119477a94640fc9465e80fd23f3cd0cc5d1_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:15,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a042f6953f894cf18b4586c27f0fbf96, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:15,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a042f6953f894cf18b4586c27f0fbf96 is 175, key is test_row_0/A:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:15,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742023_1199 (size=48489) 2024-11-19T04:41:15,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a042f6953f894cf18b4586c27f0fbf96 2024-11-19T04:41:15,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/4cdba0c9c4a44e1cb0ae58d88ddca805 is 50, key is test_row_0/B:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:15,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T04:41:15,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:15,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:15,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:15,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T04:41:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742024_1200 (size=12251) 2024-11-19T04:41:16,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T04:41:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991336127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991336127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991336127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991336127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T04:41:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T04:41:16,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/4cdba0c9c4a44e1cb0ae58d88ddca805 2024-11-19T04:41:16,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/e79a21f27107469bb34f030178cbb03f is 50, key is test_row_0/C:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:16,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742025_1201 (size=12251) 2024-11-19T04:41:16,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/e79a21f27107469bb34f030178cbb03f 2024-11-19T04:41:16,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/a042f6953f894cf18b4586c27f0fbf96 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96 2024-11-19T04:41:16,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T04:41:16,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:16,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:16,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96, entries=250, sequenceid=259, filesize=47.4 K 2024-11-19T04:41:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/4cdba0c9c4a44e1cb0ae58d88ddca805 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/4cdba0c9c4a44e1cb0ae58d88ddca805 2024-11-19T04:41:16,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/4cdba0c9c4a44e1cb0ae58d88ddca805, entries=150, sequenceid=259, filesize=12.0 K 2024-11-19T04:41:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/e79a21f27107469bb34f030178cbb03f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e79a21f27107469bb34f030178cbb03f 2024-11-19T04:41:16,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e79a21f27107469bb34f030178cbb03f, entries=150, sequenceid=259, filesize=12.0 K 2024-11-19T04:41:16,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 2579823445e39a7eaea2aba4ffb4cf67 in 970ms, sequenceid=259, compaction requested=true 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:16,409 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:16,409 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:16,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:16,411 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:16,411 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:16,411 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:16,411 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:16,411 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,411 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,411 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/0166d2f5b48f486da4fcc7917b49600a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=108.6 K 2024-11-19T04:41:16,411 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/995870c67a7a445b8944e4acf45b5f72, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/07b01763e336452f9f4369b0b1cfe242, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/4cdba0c9c4a44e1cb0ae58d88ddca805] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.2 K 2024-11-19T04:41:16,411 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,411 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/0166d2f5b48f486da4fcc7917b49600a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96] 2024-11-19T04:41:16,411 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 995870c67a7a445b8944e4acf45b5f72, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731991273425 2024-11-19T04:41:16,412 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0166d2f5b48f486da4fcc7917b49600a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731991273425 2024-11-19T04:41:16,412 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b01763e336452f9f4369b0b1cfe242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991273618 2024-11-19T04:41:16,412 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cdba0c9c4a44e1cb0ae58d88ddca805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991274797 2024-11-19T04:41:16,412 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ce5598e97649d4a7b910453147761f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991273618 2024-11-19T04:41:16,413 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a042f6953f894cf18b4586c27f0fbf96, keycount=250, bloomtype=ROW, size=47.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991274776 2024-11-19T04:41:16,425 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:16,431 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:16,431 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1a8f226f07024d3b954f69dd250cc00b is 50, key is test_row_0/B:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:16,437 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411193fec6936a1b943acbcc2f251bbae616b_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:16,439 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411193fec6936a1b943acbcc2f251bbae616b_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:16,439 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193fec6936a1b943acbcc2f251bbae616b_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:16,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742026_1202 (size=12865) 2024-11-19T04:41:16,464 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1a8f226f07024d3b954f69dd250cc00b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1a8f226f07024d3b954f69dd250cc00b 2024-11-19T04:41:16,474 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 1a8f226f07024d3b954f69dd250cc00b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:16,474 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:16,474 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991276409; duration=0sec 2024-11-19T04:41:16,474 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:16,474 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:16,474 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:16,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742027_1203 (size=4469) 2024-11-19T04:41:16,478 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#168 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:16,479 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/c26fdf5e9e7e4414ab92ea738e6dd6c7 is 175, key is test_row_0/A:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:16,479 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:16,479 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:16,479 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,480 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/753bafeb219f43b39ebc041eca5c5233, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4681891d78f5467a924f3576de159e58, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e79a21f27107469bb34f030178cbb03f] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.2 K 2024-11-19T04:41:16,481 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 753bafeb219f43b39ebc041eca5c5233, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731991273425 2024-11-19T04:41:16,481 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 4681891d78f5467a924f3576de159e58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991273618 2024-11-19T04:41:16,482 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e79a21f27107469bb34f030178cbb03f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991274797 2024-11-19T04:41:16,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742028_1204 (size=31819) 2024-11-19T04:41:16,507 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:16,507 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/9ffdcce9769a4e3d85865038341ff301 is 50, key is test_row_0/C:col10/1731991274797/Put/seqid=0 2024-11-19T04:41:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742029_1205 (size=12865) 2024-11-19T04:41:16,538 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/9ffdcce9769a4e3d85865038341ff301 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ffdcce9769a4e3d85865038341ff301 2024-11-19T04:41:16,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T04:41:16,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:16,540 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:41:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:16,542 INFO [master/08a7f35e60d4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T04:41:16,542 INFO [master/08a7f35e60d4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T04:41:16,546 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 9ffdcce9769a4e3d85865038341ff301(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:16,546 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:16,546 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991276409; duration=0sec 2024-11-19T04:41:16,546 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:16,546 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c02a992aea9f4c00b843c951bf631c97_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991275493/Put/seqid=0 2024-11-19T04:41:16,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742030_1206 (size=12454) 2024-11-19T04:41:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:16,573 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c02a992aea9f4c00b843c951bf631c97_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c02a992aea9f4c00b843c951bf631c97_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/de76a72547b84fa8ac6ee5c24004605b, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:16,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/de76a72547b84fa8ac6ee5c24004605b is 175, key is test_row_0/A:col10/1731991275493/Put/seqid=0 2024-11-19T04:41:16,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742031_1207 (size=31255) 2024-11-19T04:41:16,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:16,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991336685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991336686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991336693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991336694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T04:41:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991336797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991336798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991336807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991336807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:16,913 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/c26fdf5e9e7e4414ab92ea738e6dd6c7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/c26fdf5e9e7e4414ab92ea738e6dd6c7 2024-11-19T04:41:16,920 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into c26fdf5e9e7e4414ab92ea738e6dd6c7(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:16,921 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:16,921 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991276409; duration=0sec 2024-11-19T04:41:16,921 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:16,921 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:16,999 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/de76a72547b84fa8ac6ee5c24004605b 2024-11-19T04:41:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991337007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/dec49e54e32649e1b54913d4b5a4d779 is 50, key is test_row_0/B:col10/1731991275493/Put/seqid=0 2024-11-19T04:41:17,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991337021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991337021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991337021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742032_1208 (size=12301) 2024-11-19T04:41:17,076 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/dec49e54e32649e1b54913d4b5a4d779 2024-11-19T04:41:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/9ecfc3affce945f29425bc4e07a35f9e is 50, key is test_row_0/C:col10/1731991275493/Put/seqid=0 2024-11-19T04:41:17,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742033_1209 (size=12301) 2024-11-19T04:41:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991337317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991337327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991337338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991337338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,508 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/9ecfc3affce945f29425bc4e07a35f9e 2024-11-19T04:41:17,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/de76a72547b84fa8ac6ee5c24004605b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b 2024-11-19T04:41:17,523 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b, entries=150, sequenceid=280, filesize=30.5 K 2024-11-19T04:41:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/dec49e54e32649e1b54913d4b5a4d779 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dec49e54e32649e1b54913d4b5a4d779 2024-11-19T04:41:17,530 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dec49e54e32649e1b54913d4b5a4d779, entries=150, sequenceid=280, filesize=12.0 K 2024-11-19T04:41:17,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/9ecfc3affce945f29425bc4e07a35f9e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ecfc3affce945f29425bc4e07a35f9e 2024-11-19T04:41:17,537 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ecfc3affce945f29425bc4e07a35f9e, entries=150, sequenceid=280, filesize=12.0 K 2024-11-19T04:41:17,539 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2579823445e39a7eaea2aba4ffb4cf67 in 999ms, sequenceid=280, compaction requested=false 2024-11-19T04:41:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T04:41:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T04:41:17,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T04:41:17,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-19T04:41:17,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9340 sec 2024-11-19T04:41:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T04:41:17,757 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:17,758 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-19T04:41:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:17,760 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:17,762 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:17,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:17,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:41:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:17,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193056862a7d534fcaa794c446cff4e825_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:17,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742034_1210 (size=14994) 2024-11-19T04:41:17,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:17,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193056862a7d534fcaa794c446cff4e825_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193056862a7d534fcaa794c446cff4e825_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:17,863 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/ac066c8263b442589d072121888466dd, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:17,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/ac066c8263b442589d072121888466dd is 175, key is test_row_0/A:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:17,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991337858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:17,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991337862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991337863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991337869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991337870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742035_1211 (size=39949) 2024-11-19T04:41:17,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/ac066c8263b442589d072121888466dd 2024-11-19T04:41:17,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/86eea044981543e7a54a8c5be3b2e0eb is 50, key is test_row_0/B:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:17,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T04:41:17,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:17,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:17,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:17,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:17,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742036_1212 (size=12301) 2024-11-19T04:41:17,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991337977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991337977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991337978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991337987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991337987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:17,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T04:41:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991338187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991338187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991338187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991338197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991338197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T04:41:18,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:18,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/86eea044981543e7a54a8c5be3b2e0eb 2024-11-19T04:41:18,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/6023ed92e70446798acaa4f5b83b56e3 is 50, key is test_row_0/C:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742037_1213 (size=12301) 2024-11-19T04:41:18,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/6023ed92e70446798acaa4f5b83b56e3 2024-11-19T04:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/ac066c8263b442589d072121888466dd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd 2024-11-19T04:41:18,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd, entries=200, sequenceid=295, filesize=39.0 K 2024-11-19T04:41:18,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/86eea044981543e7a54a8c5be3b2e0eb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/86eea044981543e7a54a8c5be3b2e0eb 2024-11-19T04:41:18,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/86eea044981543e7a54a8c5be3b2e0eb, entries=150, sequenceid=295, filesize=12.0 K 2024-11-19T04:41:18,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T04:41:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/6023ed92e70446798acaa4f5b83b56e3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/6023ed92e70446798acaa4f5b83b56e3 2024-11-19T04:41:18,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/6023ed92e70446798acaa4f5b83b56e3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-19T04:41:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:18,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2579823445e39a7eaea2aba4ffb4cf67 in 565ms, sequenceid=295, compaction requested=true 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:18,394 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:18,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:18,394 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:18,397 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:18,397 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:18,397 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,397 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1a8f226f07024d3b954f69dd250cc00b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dec49e54e32649e1b54913d4b5a4d779, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/86eea044981543e7a54a8c5be3b2e0eb] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.6 K 2024-11-19T04:41:18,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:18,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:18,398 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,398 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/c26fdf5e9e7e4414ab92ea738e6dd6c7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=100.6 K 2024-11-19T04:41:18,398 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/c26fdf5e9e7e4414ab92ea738e6dd6c7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd] 2024-11-19T04:41:18,399 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c26fdf5e9e7e4414ab92ea738e6dd6c7, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991274797 2024-11-19T04:41:18,399 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a8f226f07024d3b954f69dd250cc00b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991274797 2024-11-19T04:41:18,399 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting dec49e54e32649e1b54913d4b5a4d779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731991275474 2024-11-19T04:41:18,399 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting de76a72547b84fa8ac6ee5c24004605b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731991275474 2024-11-19T04:41:18,400 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac066c8263b442589d072121888466dd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731991276656 2024-11-19T04:41:18,400 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 86eea044981543e7a54a8c5be3b2e0eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731991276656 2024-11-19T04:41:18,417 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:18,422 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197a2805519e774d75ae754def746f9214_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:18,423 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:18,424 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/9376a20f243d44698c16fad4a09b31bb is 50, key is test_row_0/B:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:18,430 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197a2805519e774d75ae754def746f9214_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:18,430 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a2805519e774d75ae754def746f9214_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:18,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742038_1214 (size=13017) 2024-11-19T04:41:18,471 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/9376a20f243d44698c16fad4a09b31bb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/9376a20f243d44698c16fad4a09b31bb 2024-11-19T04:41:18,488 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 9376a20f243d44698c16fad4a09b31bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:18,488 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:18,488 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991278394; duration=0sec 2024-11-19T04:41:18,488 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:18,488 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:18,488 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742039_1215 (size=4469) 2024-11-19T04:41:18,491 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:18,491 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:18,492 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,492 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ffdcce9769a4e3d85865038341ff301, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ecfc3affce945f29425bc4e07a35f9e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/6023ed92e70446798acaa4f5b83b56e3] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.6 K 2024-11-19T04:41:18,492 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ffdcce9769a4e3d85865038341ff301, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991274797 2024-11-19T04:41:18,493 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ecfc3affce945f29425bc4e07a35f9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731991275474 2024-11-19T04:41:18,493 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6023ed92e70446798acaa4f5b83b56e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731991276656 2024-11-19T04:41:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:18,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T04:41:18,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991277862, max timestamp=1731991278501Num uniques 859; 2024-11-19T04:41:18,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:18,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991277862, max timestamp=1731991278501Num uniques 859; 2024-11-19T04:41:18,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:18,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991277862, max timestamp=1731991278501Num uniques 859; 2024-11-19T04:41:18,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:18,510 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:18,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:18,510 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/2b661a90cf5f4735b675e8c1c07c701d is 50, key is test_row_0/C:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:18,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111925f77981e43a4dd0b0308c7e7245b4e7_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991277862/Put/seqid=0 2024-11-19T04:41:18,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742040_1216 (size=13017) 2024-11-19T04:41:18,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T04:41:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742041_1217 (size=12454) 2024-11-19T04:41:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:18,552 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:18,557 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111925f77981e43a4dd0b0308c7e7245b4e7_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111925f77981e43a4dd0b0308c7e7245b4e7_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:18,558 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b1ba5a6f4d4f4a568f1cfae65669c074, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:18,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b1ba5a6f4d4f4a568f1cfae65669c074 is 175, key is test_row_0/A:col10/1731991277862/Put/seqid=0 2024-11-19T04:41:18,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731991278530, max timestamp=1731991278552Num uniques 859; 2024-11-19T04:41:18,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T04:41:18,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731991278530, max timestamp=1731991278552Num uniques 859; 2024-11-19T04:41:18,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731991278530, max timestamp=1731991278552Num uniques 859; 2024-11-19T04:41:18,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T04:41:18,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-19T04:41:18,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742042_1218 (size=31255) 2024-11-19T04:41:18,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b1ba5a6f4d4f4a568f1cfae65669c074 2024-11-19T04:41:18,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/dca1ec93416e4e44896434ae3809c781 is 50, key is test_row_0/B:col10/1731991277862/Put/seqid=0 2024-11-19T04:41:18,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991338592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991338595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991338600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991338604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742043_1219 (size=12301) 2024-11-19T04:41:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/dca1ec93416e4e44896434ae3809c781 2024-11-19T04:41:18,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/4ea96a592adc40b9a8ad18845feac513 is 50, key is test_row_0/C:col10/1731991277862/Put/seqid=0 2024-11-19T04:41:18,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991338607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742044_1220 (size=12301) 2024-11-19T04:41:18,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/4ea96a592adc40b9a8ad18845feac513 2024-11-19T04:41:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b1ba5a6f4d4f4a568f1cfae65669c074 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074 2024-11-19T04:41:18,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074, entries=150, sequenceid=319, filesize=30.5 K 2024-11-19T04:41:18,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/dca1ec93416e4e44896434ae3809c781 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dca1ec93416e4e44896434ae3809c781 2024-11-19T04:41:18,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dca1ec93416e4e44896434ae3809c781, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T04:41:18,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/4ea96a592adc40b9a8ad18845feac513 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4ea96a592adc40b9a8ad18845feac513 2024-11-19T04:41:18,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4ea96a592adc40b9a8ad18845feac513, entries=150, sequenceid=319, filesize=12.0 K 2024-11-19T04:41:18,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=154.31 KB/158010 for 2579823445e39a7eaea2aba4ffb4cf67 in 187ms, sequenceid=319, compaction requested=false 2024-11-19T04:41:18,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:18,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-19T04:41:18,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:18,706 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-19T04:41:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f984417dd4c49b4a8becaf406cd7c5d_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991278600/Put/seqid=0 2024-11-19T04:41:18,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742045_1221 (size=12454) 2024-11-19T04:41:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991338723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991338729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991338737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991338741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991338747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991338847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991338848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991338857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991338857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,891 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#177 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:18,892 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/4fa6a96ca9ed449c9888a97f7d858792 is 175, key is test_row_0/A:col10/1731991276691/Put/seqid=0 2024-11-19T04:41:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742046_1222 (size=31971) 2024-11-19T04:41:18,910 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/4fa6a96ca9ed449c9888a97f7d858792 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/4fa6a96ca9ed449c9888a97f7d858792 2024-11-19T04:41:18,920 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into 4fa6a96ca9ed449c9888a97f7d858792(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:18,920 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:18,920 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991278394; duration=0sec 2024-11-19T04:41:18,920 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:18,920 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:18,956 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/2b661a90cf5f4735b675e8c1c07c701d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/2b661a90cf5f4735b675e8c1c07c701d 2024-11-19T04:41:18,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991338957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:18,972 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 2b661a90cf5f4735b675e8c1c07c701d(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:18,972 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:18,972 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991278394; duration=0sec 2024-11-19T04:41:18,972 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:18,972 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991339057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991339057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991339067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991339067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:19,144 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192f984417dd4c49b4a8becaf406cd7c5d_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f984417dd4c49b4a8becaf406cd7c5d_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/14e5743f5151455aae6cdbeb7dfdb313, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/14e5743f5151455aae6cdbeb7dfdb313 is 175, key is test_row_0/A:col10/1731991278600/Put/seqid=0 2024-11-19T04:41:19,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742047_1223 (size=31255) 2024-11-19T04:41:19,161 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/14e5743f5151455aae6cdbeb7dfdb313 2024-11-19T04:41:19,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8e2aa0a694a2400cbac02fb42b641318 is 50, key is test_row_0/B:col10/1731991278600/Put/seqid=0 2024-11-19T04:41:19,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742048_1224 (size=12301) 2024-11-19T04:41:19,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991339267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991339367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991339367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991339377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991339386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,601 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8e2aa0a694a2400cbac02fb42b641318 2024-11-19T04:41:19,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/8a4f455df4ae417a8f26e17d3d0d12f5 is 50, key is test_row_0/C:col10/1731991278600/Put/seqid=0 2024-11-19T04:41:19,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742049_1225 (size=12301) 2024-11-19T04:41:19,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991339777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991339877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991339877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991339897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991339897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:19,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:20,028 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/8a4f455df4ae417a8f26e17d3d0d12f5 2024-11-19T04:41:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/14e5743f5151455aae6cdbeb7dfdb313 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313 2024-11-19T04:41:20,051 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313, entries=150, sequenceid=355, filesize=30.5 K 2024-11-19T04:41:20,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8e2aa0a694a2400cbac02fb42b641318 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8e2aa0a694a2400cbac02fb42b641318 2024-11-19T04:41:20,064 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8e2aa0a694a2400cbac02fb42b641318, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T04:41:20,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/8a4f455df4ae417a8f26e17d3d0d12f5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/8a4f455df4ae417a8f26e17d3d0d12f5 2024-11-19T04:41:20,074 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/8a4f455df4ae417a8f26e17d3d0d12f5, entries=150, sequenceid=355, filesize=12.0 K 2024-11-19T04:41:20,077 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=40.25 KB/41220 for 2579823445e39a7eaea2aba4ffb4cf67 in 1371ms, sequenceid=355, compaction requested=true 2024-11-19T04:41:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:20,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T04:41:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T04:41:20,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T04:41:20,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-11-19T04:41:20,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.3240 sec 2024-11-19T04:41:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:20,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:41:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:20,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:20,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:20,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111993f30f58ff80482a927373927f6cabce_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:20,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742050_1226 (size=14994) 2024-11-19T04:41:20,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991280829, max timestamp=1731991280879Num uniques 859; 2024-11-19T04:41:20,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991280829, max timestamp=1731991280879Num uniques 859; 2024-11-19T04:41:20,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991280829, max timestamp=1731991280879Num uniques 859; 2024-11-19T04:41:20,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:20,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:20,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991340912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991340913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991340914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991340917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991340918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:20,919 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991341026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991341027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991341027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991341027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,214 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:21,218 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111993f30f58ff80482a927373927f6cabce_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111993f30f58ff80482a927373927f6cabce_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:21,219 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/99dad183344d4e48a7cbadc2202c5f6b, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:21,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/99dad183344d4e48a7cbadc2202c5f6b is 175, key is test_row_0/A:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:21,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742051_1227 (size=39949) 2024-11-19T04:41:21,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991341237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991341247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991341247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991341247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991341546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991341557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:21,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/99dad183344d4e48a7cbadc2202c5f6b 2024-11-19T04:41:21,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f57225d7bf774ec5850c2082c90ac586 is 50, key is test_row_0/B:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:21,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742052_1228 (size=12301) 2024-11-19T04:41:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T04:41:21,917 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:21,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-19T04:41:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T04:41:21,920 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:21,921 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:21,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T04:41:22,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f57225d7bf774ec5850c2082c90ac586 2024-11-19T04:41:22,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/5de4fa364cfe43aa9bb58f049b35b987 is 50, key is test_row_0/C:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:22,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742053_1229 (size=12301) 2024-11-19T04:41:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991342057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:22,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T04:41:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991342077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991342077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:22,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991342077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:22,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T04:41:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T04:41:22,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T04:41:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:22,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/5de4fa364cfe43aa9bb58f049b35b987 2024-11-19T04:41:22,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/99dad183344d4e48a7cbadc2202c5f6b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b 2024-11-19T04:41:22,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b, entries=200, sequenceid=367, filesize=39.0 K 2024-11-19T04:41:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f57225d7bf774ec5850c2082c90ac586 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f57225d7bf774ec5850c2082c90ac586 2024-11-19T04:41:22,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f57225d7bf774ec5850c2082c90ac586, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T04:41:22,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/5de4fa364cfe43aa9bb58f049b35b987 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5de4fa364cfe43aa9bb58f049b35b987 2024-11-19T04:41:22,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5de4fa364cfe43aa9bb58f049b35b987, entries=150, sequenceid=367, filesize=12.0 K 2024-11-19T04:41:22,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2579823445e39a7eaea2aba4ffb4cf67 in 1672ms, sequenceid=367, compaction requested=true 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:22,472 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:22,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:22,472 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:22,474 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:22,474 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:22,474 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,475 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/4fa6a96ca9ed449c9888a97f7d858792, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=131.3 K 2024-11-19T04:41:22,475 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,475 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/4fa6a96ca9ed449c9888a97f7d858792, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b] 2024-11-19T04:41:22,475 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:22,475 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:22,475 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,475 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/9376a20f243d44698c16fad4a09b31bb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dca1ec93416e4e44896434ae3809c781, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8e2aa0a694a2400cbac02fb42b641318, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f57225d7bf774ec5850c2082c90ac586] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=48.8 K 2024-11-19T04:41:22,476 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa6a96ca9ed449c9888a97f7d858792, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731991276656 2024-11-19T04:41:22,476 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9376a20f243d44698c16fad4a09b31bb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731991276656 2024-11-19T04:41:22,476 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting dca1ec93416e4e44896434ae3809c781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731991277862 2024-11-19T04:41:22,476 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ba5a6f4d4f4a568f1cfae65669c074, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731991277862 2024-11-19T04:41:22,476 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2aa0a694a2400cbac02fb42b641318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731991278592 2024-11-19T04:41:22,477 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e5743f5151455aae6cdbeb7dfdb313, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731991278592 2024-11-19T04:41:22,477 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f57225d7bf774ec5850c2082c90ac586, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731991278716 2024-11-19T04:41:22,477 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99dad183344d4e48a7cbadc2202c5f6b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731991278716 2024-11-19T04:41:22,488 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:22,489 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1d95058aad8a4170aabada0c4136f1ff is 50, key is test_row_0/B:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:22,493 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:22,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742054_1230 (size=13153) 2024-11-19T04:41:22,495 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195010f268d3434979bccc036f9750bfdf_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:22,498 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195010f268d3434979bccc036f9750bfdf_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:22,498 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195010f268d3434979bccc036f9750bfdf_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:22,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742055_1231 (size=4469) 2024-11-19T04:41:22,504 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#190 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:22,505 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/339477b4cb4f46e6921a387fe4694a36 is 175, key is test_row_0/A:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:22,505 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/1d95058aad8a4170aabada0c4136f1ff as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1d95058aad8a4170aabada0c4136f1ff 2024-11-19T04:41:22,511 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 1d95058aad8a4170aabada0c4136f1ff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:22,511 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:22,511 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=12, startTime=1731991282472; duration=0sec 2024-11-19T04:41:22,511 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:22,512 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:22,512 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:22,513 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:22,513 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:22,513 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,513 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/2b661a90cf5f4735b675e8c1c07c701d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4ea96a592adc40b9a8ad18845feac513, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/8a4f455df4ae417a8f26e17d3d0d12f5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5de4fa364cfe43aa9bb58f049b35b987] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=48.8 K 2024-11-19T04:41:22,514 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b661a90cf5f4735b675e8c1c07c701d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731991276656 2024-11-19T04:41:22,514 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea96a592adc40b9a8ad18845feac513, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731991277862 2024-11-19T04:41:22,515 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4f455df4ae417a8f26e17d3d0d12f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731991278592 2024-11-19T04:41:22,515 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de4fa364cfe43aa9bb58f049b35b987, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731991278716 2024-11-19T04:41:22,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742056_1232 (size=32107) 2024-11-19T04:41:22,530 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:22,530 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/aaeed791592047109d5d96aab5f70a93 is 50, key is test_row_0/C:col10/1731991278731/Put/seqid=0 2024-11-19T04:41:22,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-19T04:41:22,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:22,532 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T04:41:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119478907c15b52471e8d1ddc34bf658539_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991280900/Put/seqid=0 2024-11-19T04:41:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T04:41:22,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742057_1233 (size=13153) 2024-11-19T04:41:22,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742058_1234 (size=12454) 2024-11-19T04:41:22,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:22,561 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119478907c15b52471e8d1ddc34bf658539_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119478907c15b52471e8d1ddc34bf658539_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:22,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/81e8360a647f4495ae2f8d80fc197365, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:22,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/81e8360a647f4495ae2f8d80fc197365 is 175, key is test_row_0/A:col10/1731991280900/Put/seqid=0 2024-11-19T04:41:22,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742059_1235 (size=31255) 2024-11-19T04:41:22,930 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/339477b4cb4f46e6921a387fe4694a36 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/339477b4cb4f46e6921a387fe4694a36 2024-11-19T04:41:22,937 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into 339477b4cb4f46e6921a387fe4694a36(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:22,937 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:22,937 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=12, startTime=1731991282472; duration=0sec 2024-11-19T04:41:22,938 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:22,938 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:22,960 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/aaeed791592047109d5d96aab5f70a93 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/aaeed791592047109d5d96aab5f70a93 2024-11-19T04:41:22,965 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into aaeed791592047109d5d96aab5f70a93(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:22,965 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:22,965 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=12, startTime=1731991282472; duration=0sec 2024-11-19T04:41:22,965 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:22,965 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:22,971 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/81e8360a647f4495ae2f8d80fc197365 2024-11-19T04:41:22,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8b9dde4b9a2648af8b8a9e0730d7e0f8 is 50, key is test_row_0/B:col10/1731991280900/Put/seqid=0 2024-11-19T04:41:22,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742060_1236 (size=12301) 2024-11-19T04:41:22,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991342985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T04:41:23,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991343068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,073 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991343097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991343097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991343097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991343097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,098 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,098 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,099 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991343307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,384 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8b9dde4b9a2648af8b8a9e0730d7e0f8 2024-11-19T04:41:23,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b386836b8c7c4dfb8ba64537982942fa is 50, key is test_row_0/C:col10/1731991280900/Put/seqid=0 2024-11-19T04:41:23,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742061_1237 (size=12301) 2024-11-19T04:41:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991343616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:23,797 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b386836b8c7c4dfb8ba64537982942fa 2024-11-19T04:41:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/81e8360a647f4495ae2f8d80fc197365 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365 2024-11-19T04:41:23,806 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365, entries=150, sequenceid=404, filesize=30.5 K 2024-11-19T04:41:23,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/8b9dde4b9a2648af8b8a9e0730d7e0f8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8b9dde4b9a2648af8b8a9e0730d7e0f8 2024-11-19T04:41:23,812 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8b9dde4b9a2648af8b8a9e0730d7e0f8, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T04:41:23,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b386836b8c7c4dfb8ba64537982942fa as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b386836b8c7c4dfb8ba64537982942fa 2024-11-19T04:41:23,817 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b386836b8c7c4dfb8ba64537982942fa, entries=150, sequenceid=404, filesize=12.0 K 2024-11-19T04:41:23,817 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 2579823445e39a7eaea2aba4ffb4cf67 in 1285ms, sequenceid=404, compaction requested=false 2024-11-19T04:41:23,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:23,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:23,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-19T04:41:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-19T04:41:23,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T04:41:23,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-19T04:41:23,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9050 sec 2024-11-19T04:41:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T04:41:24,067 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:24,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-19T04:41:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T04:41:24,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:24,070 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:24,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:24,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:41:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:24,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:24,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e5415de76054a51875672ba58dcb81c_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742062_1238 (size=14994) 2024-11-19T04:41:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T04:41:24,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991284173, max timestamp=1731991284210Num uniques 859; 2024-11-19T04:41:24,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991284173, max timestamp=1731991284210Num uniques 859; 2024-11-19T04:41:24,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:24,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991284173, max timestamp=1731991284210Num uniques 859; 2024-11-19T04:41:24,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:24,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:24,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991344288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:24,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:24,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T04:41:24,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:24,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:24,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:24,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,544 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:24,549 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e5415de76054a51875672ba58dcb81c_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e5415de76054a51875672ba58dcb81c_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:24,550 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/75d1e33836f5412cb2bbd8332a5ec04d, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:24,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/75d1e33836f5412cb2bbd8332a5ec04d is 175, key is test_row_0/A:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:24,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742063_1239 (size=39949) 2024-11-19T04:41:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991344606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:24,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:24,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:24,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T04:41:24,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:24,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:24,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991344917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:24,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/75d1e33836f5412cb2bbd8332a5ec04d 2024-11-19T04:41:24,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f73d34ccf7f24a5b906313f5f2f800eb is 50, key is test_row_0/B:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:24,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742064_1240 (size=12301) 2024-11-19T04:41:24,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:24,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:24,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:24,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731991345087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:25,091 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42912 deadline: 1731991345107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:25,111 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42898 deadline: 1731991345126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42870 deadline: 1731991345127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:25,130 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:25,130 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 2579823445e39a7eaea2aba4ffb4cf67 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:25,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T04:41:25,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:25,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:25,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:25,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f73d34ccf7f24a5b906313f5f2f800eb 2024-11-19T04:41:25,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/eb1a534010b34920b7be72a2e3847a73 is 50, key is test_row_0/C:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:25,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742065_1241 (size=12301) 2024-11-19T04:41:25,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/eb1a534010b34920b7be72a2e3847a73 2024-11-19T04:41:25,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/75d1e33836f5412cb2bbd8332a5ec04d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d 2024-11-19T04:41:25,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d, entries=200, sequenceid=417, filesize=39.0 K 2024-11-19T04:41:25,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/f73d34ccf7f24a5b906313f5f2f800eb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f73d34ccf7f24a5b906313f5f2f800eb 2024-11-19T04:41:25,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f73d34ccf7f24a5b906313f5f2f800eb, entries=150, sequenceid=417, filesize=12.0 K 2024-11-19T04:41:25,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991345427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:25,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/eb1a534010b34920b7be72a2e3847a73 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/eb1a534010b34920b7be72a2e3847a73 2024-11-19T04:41:25,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:25,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/eb1a534010b34920b7be72a2e3847a73, entries=150, sequenceid=417, filesize=12.0 K 2024-11-19T04:41:25,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2579823445e39a7eaea2aba4ffb4cf67 in 1306ms, sequenceid=417, compaction requested=true 2024-11-19T04:41:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:25,436 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:25,437 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:25,437 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:25,437 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:25,438 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,438 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:25,438 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/339477b4cb4f46e6921a387fe4694a36, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=100.9 K 2024-11-19T04:41:25,438 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:25,438 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,438 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,438 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/339477b4cb4f46e6921a387fe4694a36, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d] 2024-11-19T04:41:25,438 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1d95058aad8a4170aabada0c4136f1ff, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8b9dde4b9a2648af8b8a9e0730d7e0f8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f73d34ccf7f24a5b906313f5f2f800eb] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.9 K 2024-11-19T04:41:25,438 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339477b4cb4f46e6921a387fe4694a36, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731991278716 2024-11-19T04:41:25,438 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d95058aad8a4170aabada0c4136f1ff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731991278716 2024-11-19T04:41:25,439 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e8360a647f4495ae2f8d80fc197365, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731991280900 2024-11-19T04:41:25,439 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b9dde4b9a2648af8b8a9e0730d7e0f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731991280900 2024-11-19T04:41:25,439 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75d1e33836f5412cb2bbd8332a5ec04d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731991282963 2024-11-19T04:41:25,439 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f73d34ccf7f24a5b906313f5f2f800eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731991282963 2024-11-19T04:41:25,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-19T04:41:25,451 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:25,451 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/6018720f6a7f46088e0438bd826a9870 is 50, key is test_row_0/B:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,452 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T04:41:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:25,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:25,457 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:25,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742066_1242 (size=13255) 2024-11-19T04:41:25,464 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119600f524408e440169825434632b4353c_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:25,466 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119600f524408e440169825434632b4353c_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:25,466 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119600f524408e440169825434632b4353c_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:25,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c9c85fdfc2f4af5b474b4d46d5cb97c_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991284267/Put/seqid=0 2024-11-19T04:41:25,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742067_1243 (size=4469) 2024-11-19T04:41:25,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742068_1244 (size=12454) 2024-11-19T04:41:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,481 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#199 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:25,482 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/13972d5cb0984d09b1c402b73c619c8b is 175, key is test_row_0/A:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:25,486 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191c9c85fdfc2f4af5b474b4d46d5cb97c_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c9c85fdfc2f4af5b474b4d46d5cb97c_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742069_1245 (size=32209) 2024-11-19T04:41:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d1e19b8023ce45f0b80ef6ef96291ad2, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d1e19b8023ce45f0b80ef6ef96291ad2 is 175, key is test_row_0/A:col10/1731991284267/Put/seqid=0 2024-11-19T04:41:25,495 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/13972d5cb0984d09b1c402b73c619c8b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/13972d5cb0984d09b1c402b73c619c8b 2024-11-19T04:41:25,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742070_1246 (size=31255) 2024-11-19T04:41:25,497 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d1e19b8023ce45f0b80ef6ef96291ad2 2024-11-19T04:41:25,502 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into 13972d5cb0984d09b1c402b73c619c8b(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:25,502 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:25,502 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991285436; duration=0sec 2024-11-19T04:41:25,503 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:25,503 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:25,503 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:25,504 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:25,504 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:25,504 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,504 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/aaeed791592047109d5d96aab5f70a93, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b386836b8c7c4dfb8ba64537982942fa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/eb1a534010b34920b7be72a2e3847a73] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=36.9 K 2024-11-19T04:41:25,504 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaeed791592047109d5d96aab5f70a93, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731991278716 2024-11-19T04:41:25,505 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b386836b8c7c4dfb8ba64537982942fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731991280900 2024-11-19T04:41:25,506 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1a534010b34920b7be72a2e3847a73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731991282963 2024-11-19T04:41:25,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/b692f1607bf54ff0933991f276735904 is 50, key is test_row_0/B:col10/1731991284267/Put/seqid=0 2024-11-19T04:41:25,524 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:25,524 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/92a77de69ab040a1acf715a554419740 is 50, key is test_row_0/C:col10/1731991282981/Put/seqid=0 2024-11-19T04:41:25,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742071_1247 (size=12301) 2024-11-19T04:41:25,531 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/b692f1607bf54ff0933991f276735904 2024-11-19T04:41:25,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742072_1248 (size=13255) 2024-11-19T04:41:25,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/34da0d9a411b4936b26b0dcc15d05888 is 50, key is test_row_0/C:col10/1731991284267/Put/seqid=0 2024-11-19T04:41:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742073_1249 (size=12301) 2024-11-19T04:41:25,552 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/92a77de69ab040a1acf715a554419740 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/92a77de69ab040a1acf715a554419740 2024-11-19T04:41:25,561 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into 92a77de69ab040a1acf715a554419740(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:25,561 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:25,561 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=13, startTime=1731991285437; duration=0sec 2024-11-19T04:41:25,562 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:25,562 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:25,871 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/6018720f6a7f46088e0438bd826a9870 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/6018720f6a7f46088e0438bd826a9870 2024-11-19T04:41:25,876 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 6018720f6a7f46088e0438bd826a9870(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:25,876 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:25,876 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991285437; duration=0sec 2024-11-19T04:41:25,876 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:25,876 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:25,953 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/34da0d9a411b4936b26b0dcc15d05888 2024-11-19T04:41:25,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/d1e19b8023ce45f0b80ef6ef96291ad2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2 2024-11-19T04:41:25,968 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2, entries=150, sequenceid=452, filesize=30.5 K 2024-11-19T04:41:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/b692f1607bf54ff0933991f276735904 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b692f1607bf54ff0933991f276735904 2024-11-19T04:41:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,974 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b692f1607bf54ff0933991f276735904, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T04:41:25,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/34da0d9a411b4936b26b0dcc15d05888 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/34da0d9a411b4936b26b0dcc15d05888 2024-11-19T04:41:25,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,980 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/34da0d9a411b4936b26b0dcc15d05888, entries=150, sequenceid=452, filesize=12.0 K 2024-11-19T04:41:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,981 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 2579823445e39a7eaea2aba4ffb4cf67 in 529ms, sequenceid=452, compaction requested=false 2024-11-19T04:41:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-19T04:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-19T04:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T04:41:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-19T04:41:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.9200 sec 2024-11-19T04:41:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T04:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,227 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:26,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-19T04:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T04:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,230 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:26,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T04:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-19T04:41:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-19T04:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-19T04:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T04:41:26,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-19T04:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 161 msec 2024-11-19T04:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T04:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,547 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,548 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T04:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T04:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,551 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:41:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111941ec737e35564dfe9c7349419de9a762_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991286540/Put/seqid=0 2024-11-19T04:41:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742074_1250 (size=20074) 2024-11-19T04:41:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111941ec737e35564dfe9c7349419de9a762_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941ec737e35564dfe9c7349419de9a762_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/89063cf1a9a34e4a9eb832317a59d11d, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/89063cf1a9a34e4a9eb832317a59d11d is 175, key is test_row_0/A:col10/1731991286540/Put/seqid=0 2024-11-19T04:41:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:26,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742075_1251 (size=57333) 2024-11-19T04:41:26,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=466, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/89063cf1a9a34e4a9eb832317a59d11d 2024-11-19T04:41:26,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/5045850f6ab94dfb9706506211911cc0 is 50, key is test_row_0/B:col10/1731991286540/Put/seqid=0 2024-11-19T04:41:26,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742076_1252 (size=12301) 2024-11-19T04:41:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T04:41:26,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991286658, max timestamp=1731991286673Num uniques 859; 2024-11-19T04:41:26,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:26,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991286658, max timestamp=1731991286673Num uniques 859; 2024-11-19T04:41:26,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991286658, max timestamp=1731991286673Num uniques 859; 2024-11-19T04:41:26,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:26,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:26,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T04:41:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:26,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:26,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:26,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991346769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:26,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T04:41:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:26,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:26,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:26,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T04:41:26,869 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-19T04:41:26,869 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:26,869 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,869 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:26,870 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-19T04:41:26,870 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:26,870 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,870 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,870 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:26,871 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-19T04:41:26,871 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:26,871 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,871 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:26,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991346876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:26,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:26,880 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-19T04:41:26,880 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:26,880 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,880 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:26,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:27,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T04:41:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/5045850f6ab94dfb9706506211911cc0 2024-11-19T04:41:27,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5 is 50, key is test_row_0/C:col10/1731991286540/Put/seqid=0 2024-11-19T04:41:27,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742077_1253 (size=12301) 2024-11-19T04:41:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991347087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:27,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T04:41:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T04:41:27,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T04:41:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. as already flushing 2024-11-19T04:41:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731991347396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:41:27,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5 2024-11-19T04:41:27,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/89063cf1a9a34e4a9eb832317a59d11d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d 2024-11-19T04:41:27,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d, entries=300, sequenceid=466, filesize=56.0 K 2024-11-19T04:41:27,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/5045850f6ab94dfb9706506211911cc0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/5045850f6ab94dfb9706506211911cc0 2024-11-19T04:41:27,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/5045850f6ab94dfb9706506211911cc0, entries=150, sequenceid=466, filesize=12.0 K 2024-11-19T04:41:27,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5 2024-11-19T04:41:27,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5, entries=150, sequenceid=466, filesize=12.0 K 2024-11-19T04:41:27,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2579823445e39a7eaea2aba4ffb4cf67 in 886ms, sequenceid=466, compaction requested=true 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:27,463 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2579823445e39a7eaea2aba4ffb4cf67:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:27,463 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/A is initiating minor compaction (all files) 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:27,464 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/A in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/B is initiating minor compaction (all files) 2024-11-19T04:41:27,464 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/B in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,464 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/13972d5cb0984d09b1c402b73c619c8b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=118.0 K 2024-11-19T04:41:27,464 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,464 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/6018720f6a7f46088e0438bd826a9870, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b692f1607bf54ff0933991f276735904, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/5045850f6ab94dfb9706506211911cc0] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=37.0 K 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/13972d5cb0984d09b1c402b73c619c8b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d] 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6018720f6a7f46088e0438bd826a9870, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731991282963 2024-11-19T04:41:27,464 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13972d5cb0984d09b1c402b73c619c8b, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731991282963 2024-11-19T04:41:27,465 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b692f1607bf54ff0933991f276735904, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731991284267 2024-11-19T04:41:27,465 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e19b8023ce45f0b80ef6ef96291ad2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731991284267 2024-11-19T04:41:27,465 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89063cf1a9a34e4a9eb832317a59d11d, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731991286481 2024-11-19T04:41:27,465 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5045850f6ab94dfb9706506211911cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731991286540 2024-11-19T04:41:27,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T04:41:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,471 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T04:41:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:27,473 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:27,474 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/798f62003d494b29a12374895eb22ad5 is 50, key is test_row_0/B:col10/1731991286540/Put/seqid=0 2024-11-19T04:41:27,475 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:27,477 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119612c2ed512284015ac2e3875c74649e7_2579823445e39a7eaea2aba4ffb4cf67 store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:27,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742078_1254 (size=13357) 2024-11-19T04:41:27,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bcf2af05c45043e4b08b400193f2fc7f_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991286768/Put/seqid=0 2024-11-19T04:41:27,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742079_1255 (size=12454) 2024-11-19T04:41:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:27,493 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bcf2af05c45043e4b08b400193f2fc7f_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bcf2af05c45043e4b08b400193f2fc7f_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:27,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/32c6940a3785427ab66c115e57a3de19, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/32c6940a3785427ab66c115e57a3de19 is 175, key is test_row_0/A:col10/1731991286768/Put/seqid=0 2024-11-19T04:41:27,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742080_1256 (size=31255) 2024-11-19T04:41:27,500 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=503, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/32c6940a3785427ab66c115e57a3de19 2024-11-19T04:41:27,502 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119612c2ed512284015ac2e3875c74649e7_2579823445e39a7eaea2aba4ffb4cf67, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:27,502 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119612c2ed512284015ac2e3875c74649e7_2579823445e39a7eaea2aba4ffb4cf67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:27,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742081_1257 (size=4469) 2024-11-19T04:41:27,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/cf1641458a964a6393526156c8cff1f4 is 50, key is test_row_0/B:col10/1731991286768/Put/seqid=0 2024-11-19T04:41:27,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742082_1258 (size=12301) 2024-11-19T04:41:27,514 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/cf1641458a964a6393526156c8cff1f4 2024-11-19T04:41:27,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/ff1bb33208834124b2df41f05744e2de is 50, key is test_row_0/C:col10/1731991286768/Put/seqid=0 2024-11-19T04:41:27,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742083_1259 (size=12301) 2024-11-19T04:41:27,533 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/ff1bb33208834124b2df41f05744e2de 2024-11-19T04:41:27,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/32c6940a3785427ab66c115e57a3de19 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/32c6940a3785427ab66c115e57a3de19 2024-11-19T04:41:27,543 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/32c6940a3785427ab66c115e57a3de19, entries=150, sequenceid=503, filesize=30.5 K 2024-11-19T04:41:27,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/cf1641458a964a6393526156c8cff1f4 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/cf1641458a964a6393526156c8cff1f4 2024-11-19T04:41:27,548 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/cf1641458a964a6393526156c8cff1f4, entries=150, sequenceid=503, filesize=12.0 K 2024-11-19T04:41:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/ff1bb33208834124b2df41f05744e2de as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ff1bb33208834124b2df41f05744e2de 2024-11-19T04:41:27,553 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ff1bb33208834124b2df41f05744e2de, entries=150, sequenceid=503, filesize=12.0 K 2024-11-19T04:41:27,553 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 2579823445e39a7eaea2aba4ffb4cf67 in 82ms, sequenceid=503, compaction requested=true 2024-11-19T04:41:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T04:41:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T04:41:27,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T04:41:27,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-11-19T04:41:27,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.0090 sec 2024-11-19T04:41:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T04:41:27,687 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:27,884 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/798f62003d494b29a12374895eb22ad5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/798f62003d494b29a12374895eb22ad5 2024-11-19T04:41:27,889 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/B of 2579823445e39a7eaea2aba4ffb4cf67 into 798f62003d494b29a12374895eb22ad5(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:27,889 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:27,890 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/B, priority=13, startTime=1731991287463; duration=0sec 2024-11-19T04:41:27,890 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:27,890 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:B 2024-11-19T04:41:27,890 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:27,893 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:27,893 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 2579823445e39a7eaea2aba4ffb4cf67/C is initiating minor compaction (all files) 2024-11-19T04:41:27,893 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2579823445e39a7eaea2aba4ffb4cf67/C in TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:27,894 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/92a77de69ab040a1acf715a554419740, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/34da0d9a411b4936b26b0dcc15d05888, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ff1bb33208834124b2df41f05744e2de] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp, totalSize=49.0 K 2024-11-19T04:41:27,894 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 92a77de69ab040a1acf715a554419740, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731991282963 2024-11-19T04:41:27,894 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 34da0d9a411b4936b26b0dcc15d05888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731991284267 2024-11-19T04:41:27,895 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731991286540 2024-11-19T04:41:27,895 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1bb33208834124b2df41f05744e2de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731991286729 2024-11-19T04:41:27,905 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:27,905 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b5a616dd30c6497b8381b11c4d90b736 is 50, key is test_row_0/C:col10/1731991286768/Put/seqid=0 2024-11-19T04:41:27,908 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-19T04:41:27,908 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:27,908 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:27,909 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:27,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:27,909 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2579823445e39a7eaea2aba4ffb4cf67#A#compaction#208 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:27,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742084_1260 (size=13391) 2024-11-19T04:41:27,910 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/deae685a1f91449491871f579c8ba005 is 175, key is test_row_0/A:col10/1731991286540/Put/seqid=0 2024-11-19T04:41:27,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742085_1261 (size=32311) 2024-11-19T04:41:27,916 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/b5a616dd30c6497b8381b11c4d90b736 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b5a616dd30c6497b8381b11c4d90b736 2024-11-19T04:41:27,919 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/deae685a1f91449491871f579c8ba005 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/deae685a1f91449491871f579c8ba005 2024-11-19T04:41:27,921 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/C of 2579823445e39a7eaea2aba4ffb4cf67 into b5a616dd30c6497b8381b11c4d90b736(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:27,921 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:27,921 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/C, priority=12, startTime=1731991287463; duration=0sec 2024-11-19T04:41:27,921 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:27,921 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:C 2024-11-19T04:41:27,924 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2579823445e39a7eaea2aba4ffb4cf67/A of 2579823445e39a7eaea2aba4ffb4cf67 into deae685a1f91449491871f579c8ba005(size=31.6 K), total size for store is 62.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:27,924 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2579823445e39a7eaea2aba4ffb4cf67: 2024-11-19T04:41:27,924 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., storeName=2579823445e39a7eaea2aba4ffb4cf67/A, priority=13, startTime=1731991287463; duration=0sec 2024-11-19T04:41:27,924 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:27,924 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2579823445e39a7eaea2aba4ffb4cf67:A 2024-11-19T04:41:29,128 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-19T04:41:29,128 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-19T04:41:29,128 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:29,128 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:29,128 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,128 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,129 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,129 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:29,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:29,148 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-19T04:41:29,148 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-19T04:41:29,148 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:29,148 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:29,148 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,148 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,149 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,149 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:29,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2472 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2518 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1114 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3336 rows 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1129 2024-11-19T04:41:29,149 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3378 rows 2024-11-19T04:41:29,149 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:41:29,149 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:41:29,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:29,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:29,150 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T04:41:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T04:41:29,152 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991289152"}]},"ts":"1731991289152"} 2024-11-19T04:41:29,154 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T04:41:29,154 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T04:41:29,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:41:29,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, UNASSIGN}] 2024-11-19T04:41:29,156 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, UNASSIGN 2024-11-19T04:41:29,157 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:29,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, UNASSIGN because future has completed 2024-11-19T04:41:29,158 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:41:29,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:41:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T04:41:29,310 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] handler.UnassignRegionHandler(122): Close 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:29,310 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:41:29,310 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1722): Closing 2579823445e39a7eaea2aba4ffb4cf67, disabling compactions & flushes 2024-11-19T04:41:29,310 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:29,310 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:29,310 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. after waiting 0 ms 2024-11-19T04:41:29,310 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:29,310 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(2902): Flushing 2579823445e39a7eaea2aba4ffb4cf67 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T04:41:29,311 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=A 2024-11-19T04:41:29,311 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:29,311 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=B 2024-11-19T04:41:29,311 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:29,311 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2579823445e39a7eaea2aba4ffb4cf67, store=C 2024-11-19T04:41:29,311 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:29,316 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198053d2dc85fe427ca152f47908f9cd41_2579823445e39a7eaea2aba4ffb4cf67 is 50, key is test_row_0/A:col10/1731991289147/Put/seqid=0 2024-11-19T04:41:29,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742086_1262 (size=9914) 2024-11-19T04:41:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T04:41:29,720 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:29,724 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198053d2dc85fe427ca152f47908f9cd41_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198053d2dc85fe427ca152f47908f9cd41_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:29,724 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b33173a3b71242e7b02f6fa8e728bab3, store: [table=TestAcidGuarantees family=A region=2579823445e39a7eaea2aba4ffb4cf67] 2024-11-19T04:41:29,725 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b33173a3b71242e7b02f6fa8e728bab3 is 175, key is test_row_0/A:col10/1731991289147/Put/seqid=0 2024-11-19T04:41:29,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742087_1263 (size=22561) 2024-11-19T04:41:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T04:41:30,129 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=514, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b33173a3b71242e7b02f6fa8e728bab3 2024-11-19T04:41:30,136 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/d5ec1ffcb82f421f8202e56e44ae7586 is 50, key is test_row_0/B:col10/1731991289147/Put/seqid=0 2024-11-19T04:41:30,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742088_1264 (size=9857) 2024-11-19T04:41:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T04:41:30,539 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/d5ec1ffcb82f421f8202e56e44ae7586 2024-11-19T04:41:30,546 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f71ebf2961104e62ac9e7c295c6a5d1c is 50, key is test_row_0/C:col10/1731991289147/Put/seqid=0 2024-11-19T04:41:30,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742089_1265 (size=9857) 2024-11-19T04:41:30,950 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f71ebf2961104e62ac9e7c295c6a5d1c 2024-11-19T04:41:30,954 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/A/b33173a3b71242e7b02f6fa8e728bab3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b33173a3b71242e7b02f6fa8e728bab3 2024-11-19T04:41:30,957 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b33173a3b71242e7b02f6fa8e728bab3, entries=100, sequenceid=514, filesize=22.0 K 2024-11-19T04:41:30,958 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/B/d5ec1ffcb82f421f8202e56e44ae7586 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d5ec1ffcb82f421f8202e56e44ae7586 2024-11-19T04:41:30,961 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d5ec1ffcb82f421f8202e56e44ae7586, entries=100, sequenceid=514, filesize=9.6 K 2024-11-19T04:41:30,962 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/.tmp/C/f71ebf2961104e62ac9e7c295c6a5d1c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f71ebf2961104e62ac9e7c295c6a5d1c 2024-11-19T04:41:30,965 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f71ebf2961104e62ac9e7c295c6a5d1c, entries=100, sequenceid=514, filesize=9.6 K 2024-11-19T04:41:30,966 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2579823445e39a7eaea2aba4ffb4cf67 in 1656ms, sequenceid=514, compaction requested=true 2024-11-19T04:41:30,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1512e9fcf07947918ab72095eb0d15e0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b87db6beeb0344078d468e164887c94f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/727ea5f9ba614da68fecad1d221dfa31, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/bb34b28688d54289ba57b60af799dc8e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/0166d2f5b48f486da4fcc7917b49600a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/c26fdf5e9e7e4414ab92ea738e6dd6c7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/4fa6a96ca9ed449c9888a97f7d858792, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/339477b4cb4f46e6921a387fe4694a36, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/13972d5cb0984d09b1c402b73c619c8b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d] to archive 2024-11-19T04:41:30,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:41:30,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/f95f32bb007b4ef49ddf9f42844a2a11 2024-11-19T04:41:30,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2fdb3a90005d45f7a3d535a9c8278f26 2024-11-19T04:41:30,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1512e9fcf07947918ab72095eb0d15e0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1512e9fcf07947918ab72095eb0d15e0 2024-11-19T04:41:30,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/5b8ca380e58040f6b426f1e59c5b1491 2024-11-19T04:41:30,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/dec654b0f00945c994d59508b2909ec0 2024-11-19T04:41:30,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b87db6beeb0344078d468e164887c94f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b87db6beeb0344078d468e164887c94f 2024-11-19T04:41:30,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/274d5f8070674986a4cc17f941a652d9 2024-11-19T04:41:30,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/7a7ac4ba39d74b798221de29b11e3e36 2024-11-19T04:41:30,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/727ea5f9ba614da68fecad1d221dfa31 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/727ea5f9ba614da68fecad1d221dfa31 2024-11-19T04:41:30,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/95fb4d7b9ba24c44b1abe3f999e4c891 2024-11-19T04:41:30,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a3967760c68741c6b955b707261496f1 2024-11-19T04:41:30,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/bb34b28688d54289ba57b60af799dc8e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/bb34b28688d54289ba57b60af799dc8e 2024-11-19T04:41:30,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/1f462375f25644309c93d005a840db9a 2024-11-19T04:41:30,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/2984846e9fda4c8fbd348f2fdacca210 2024-11-19T04:41:30,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/fbbdac5d85d6448389f22425d65173d6 2024-11-19T04:41:30,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/0166d2f5b48f486da4fcc7917b49600a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/0166d2f5b48f486da4fcc7917b49600a 2024-11-19T04:41:30,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d2ce5598e97649d4a7b910453147761f 2024-11-19T04:41:30,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/a042f6953f894cf18b4586c27f0fbf96 2024-11-19T04:41:30,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/c26fdf5e9e7e4414ab92ea738e6dd6c7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/c26fdf5e9e7e4414ab92ea738e6dd6c7 2024-11-19T04:41:30,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/de76a72547b84fa8ac6ee5c24004605b 2024-11-19T04:41:30,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/ac066c8263b442589d072121888466dd 2024-11-19T04:41:30,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/4fa6a96ca9ed449c9888a97f7d858792 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/4fa6a96ca9ed449c9888a97f7d858792 2024-11-19T04:41:30,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b1ba5a6f4d4f4a568f1cfae65669c074 2024-11-19T04:41:30,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/14e5743f5151455aae6cdbeb7dfdb313 2024-11-19T04:41:30,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/99dad183344d4e48a7cbadc2202c5f6b 2024-11-19T04:41:30,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/339477b4cb4f46e6921a387fe4694a36 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/339477b4cb4f46e6921a387fe4694a36 2024-11-19T04:41:30,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/81e8360a647f4495ae2f8d80fc197365 2024-11-19T04:41:30,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/75d1e33836f5412cb2bbd8332a5ec04d 2024-11-19T04:41:30,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/13972d5cb0984d09b1c402b73c619c8b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/13972d5cb0984d09b1c402b73c619c8b 2024-11-19T04:41:30,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/d1e19b8023ce45f0b80ef6ef96291ad2 2024-11-19T04:41:30,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/89063cf1a9a34e4a9eb832317a59d11d 2024-11-19T04:41:30,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1cdd5a5d2d084fbeaab3d9c741d57d65, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/aaaff56b79cb435bac9e6966ac5c48da, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b74e4b8dac9d416a9050624eac529fb9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8f74d0da38734271956ddab48f66ab4c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/997b54ae03144ee4a799e2e2e1096832, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/16c0fea3c21a45d58cbccf32202c89b4, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/7ec4532f4e4e48fd962d88d051438a84, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/338afe58dc03481c94ac92082fb4e767, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/296a84ae064940e58ace92ab5acbe539, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1625a28354e849f794c7d3e151c67c45, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d248b0bc723341ada86e133e63bad2db, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f7a2cb1b9cf2432486ea1341061d73d0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/c5b33853e4f44571815994332366ed1e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/0fa3a82edb024e07a925e4399bf8f986, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/995870c67a7a445b8944e4acf45b5f72, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/bc817a591d7641f3826b2b7434d905b3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/07b01763e336452f9f4369b0b1cfe242, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1a8f226f07024d3b954f69dd250cc00b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/4cdba0c9c4a44e1cb0ae58d88ddca805, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dec49e54e32649e1b54913d4b5a4d779, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/9376a20f243d44698c16fad4a09b31bb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/86eea044981543e7a54a8c5be3b2e0eb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dca1ec93416e4e44896434ae3809c781, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8e2aa0a694a2400cbac02fb42b641318, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1d95058aad8a4170aabada0c4136f1ff, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f57225d7bf774ec5850c2082c90ac586, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8b9dde4b9a2648af8b8a9e0730d7e0f8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/6018720f6a7f46088e0438bd826a9870, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f73d34ccf7f24a5b906313f5f2f800eb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b692f1607bf54ff0933991f276735904, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/5045850f6ab94dfb9706506211911cc0] to archive 2024-11-19T04:41:31,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:41:31,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1cdd5a5d2d084fbeaab3d9c741d57d65 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1cdd5a5d2d084fbeaab3d9c741d57d65 2024-11-19T04:41:31,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/aaaff56b79cb435bac9e6966ac5c48da to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/aaaff56b79cb435bac9e6966ac5c48da 2024-11-19T04:41:31,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b74e4b8dac9d416a9050624eac529fb9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b74e4b8dac9d416a9050624eac529fb9 2024-11-19T04:41:31,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8f74d0da38734271956ddab48f66ab4c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8f74d0da38734271956ddab48f66ab4c 2024-11-19T04:41:31,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/997b54ae03144ee4a799e2e2e1096832 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/997b54ae03144ee4a799e2e2e1096832 2024-11-19T04:41:31,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/16c0fea3c21a45d58cbccf32202c89b4 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/16c0fea3c21a45d58cbccf32202c89b4 2024-11-19T04:41:31,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/7ec4532f4e4e48fd962d88d051438a84 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/7ec4532f4e4e48fd962d88d051438a84 2024-11-19T04:41:31,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/338afe58dc03481c94ac92082fb4e767 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/338afe58dc03481c94ac92082fb4e767 2024-11-19T04:41:31,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/296a84ae064940e58ace92ab5acbe539 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/296a84ae064940e58ace92ab5acbe539 2024-11-19T04:41:31,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1625a28354e849f794c7d3e151c67c45 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1625a28354e849f794c7d3e151c67c45 2024-11-19T04:41:31,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d248b0bc723341ada86e133e63bad2db to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d248b0bc723341ada86e133e63bad2db 2024-11-19T04:41:31,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f7a2cb1b9cf2432486ea1341061d73d0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f7a2cb1b9cf2432486ea1341061d73d0 2024-11-19T04:41:31,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/c5b33853e4f44571815994332366ed1e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/c5b33853e4f44571815994332366ed1e 2024-11-19T04:41:31,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/0fa3a82edb024e07a925e4399bf8f986 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/0fa3a82edb024e07a925e4399bf8f986 2024-11-19T04:41:31,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/995870c67a7a445b8944e4acf45b5f72 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/995870c67a7a445b8944e4acf45b5f72 2024-11-19T04:41:31,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/bc817a591d7641f3826b2b7434d905b3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/bc817a591d7641f3826b2b7434d905b3 2024-11-19T04:41:31,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/07b01763e336452f9f4369b0b1cfe242 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/07b01763e336452f9f4369b0b1cfe242 2024-11-19T04:41:31,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1a8f226f07024d3b954f69dd250cc00b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1a8f226f07024d3b954f69dd250cc00b 2024-11-19T04:41:31,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/4cdba0c9c4a44e1cb0ae58d88ddca805 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/4cdba0c9c4a44e1cb0ae58d88ddca805 2024-11-19T04:41:31,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dec49e54e32649e1b54913d4b5a4d779 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dec49e54e32649e1b54913d4b5a4d779 2024-11-19T04:41:31,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/9376a20f243d44698c16fad4a09b31bb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/9376a20f243d44698c16fad4a09b31bb 2024-11-19T04:41:31,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/86eea044981543e7a54a8c5be3b2e0eb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/86eea044981543e7a54a8c5be3b2e0eb 2024-11-19T04:41:31,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dca1ec93416e4e44896434ae3809c781 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/dca1ec93416e4e44896434ae3809c781 2024-11-19T04:41:31,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8e2aa0a694a2400cbac02fb42b641318 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8e2aa0a694a2400cbac02fb42b641318 2024-11-19T04:41:31,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1d95058aad8a4170aabada0c4136f1ff to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/1d95058aad8a4170aabada0c4136f1ff 2024-11-19T04:41:31,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f57225d7bf774ec5850c2082c90ac586 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f57225d7bf774ec5850c2082c90ac586 2024-11-19T04:41:31,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8b9dde4b9a2648af8b8a9e0730d7e0f8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/8b9dde4b9a2648af8b8a9e0730d7e0f8 2024-11-19T04:41:31,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/6018720f6a7f46088e0438bd826a9870 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/6018720f6a7f46088e0438bd826a9870 2024-11-19T04:41:31,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f73d34ccf7f24a5b906313f5f2f800eb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/f73d34ccf7f24a5b906313f5f2f800eb 2024-11-19T04:41:31,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b692f1607bf54ff0933991f276735904 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/b692f1607bf54ff0933991f276735904 2024-11-19T04:41:31,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/5045850f6ab94dfb9706506211911cc0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/5045850f6ab94dfb9706506211911cc0 2024-11-19T04:41:31,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b873f2f74d9144109417dbd08851beff, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/46cc1adb1c09457ea51b2fdc78936cc8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/1b60efce4ccf402abfbed20e53b38339, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/da8c8b3b43ee416093d1f7305be804ab, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/0f7a67d29d664699931396a53e173686, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/713230dd3d1647dcb050fbc47c5d5d50, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/87e1c45d266b4779a309dc67dc22e3e9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/44b2bc99d46a4b97bbd76b0b00d6ef6a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/85f00871e52d406dad2ff15a363e65dd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f39ef1ca120b424d8fbeef2c4d7f2941, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e1601f400a734cdaa15fc21e95f82cc5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5838c4623088426ba6d906ea708602ad, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/bd48ffca86bc44bda6b4ceef68851c81, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ec758aef7b1a42da8352ab3402140ae1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/753bafeb219f43b39ebc041eca5c5233, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/47a98436c6b94415b516d7e641543649, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4681891d78f5467a924f3576de159e58, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ffdcce9769a4e3d85865038341ff301, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e79a21f27107469bb34f030178cbb03f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ecfc3affce945f29425bc4e07a35f9e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/2b661a90cf5f4735b675e8c1c07c701d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/6023ed92e70446798acaa4f5b83b56e3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4ea96a592adc40b9a8ad18845feac513, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/8a4f455df4ae417a8f26e17d3d0d12f5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/aaeed791592047109d5d96aab5f70a93, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5de4fa364cfe43aa9bb58f049b35b987, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b386836b8c7c4dfb8ba64537982942fa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/92a77de69ab040a1acf715a554419740, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/eb1a534010b34920b7be72a2e3847a73, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/34da0d9a411b4936b26b0dcc15d05888, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ff1bb33208834124b2df41f05744e2de] to archive 2024-11-19T04:41:31,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:41:31,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b873f2f74d9144109417dbd08851beff to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b873f2f74d9144109417dbd08851beff 2024-11-19T04:41:31,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/46cc1adb1c09457ea51b2fdc78936cc8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/46cc1adb1c09457ea51b2fdc78936cc8 2024-11-19T04:41:31,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/1b60efce4ccf402abfbed20e53b38339 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/1b60efce4ccf402abfbed20e53b38339 2024-11-19T04:41:31,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/da8c8b3b43ee416093d1f7305be804ab to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/da8c8b3b43ee416093d1f7305be804ab 2024-11-19T04:41:31,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/0f7a67d29d664699931396a53e173686 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/0f7a67d29d664699931396a53e173686 2024-11-19T04:41:31,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/713230dd3d1647dcb050fbc47c5d5d50 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/713230dd3d1647dcb050fbc47c5d5d50 2024-11-19T04:41:31,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/87e1c45d266b4779a309dc67dc22e3e9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/87e1c45d266b4779a309dc67dc22e3e9 2024-11-19T04:41:31,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/44b2bc99d46a4b97bbd76b0b00d6ef6a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/44b2bc99d46a4b97bbd76b0b00d6ef6a 2024-11-19T04:41:31,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/85f00871e52d406dad2ff15a363e65dd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/85f00871e52d406dad2ff15a363e65dd 2024-11-19T04:41:31,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f39ef1ca120b424d8fbeef2c4d7f2941 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f39ef1ca120b424d8fbeef2c4d7f2941 2024-11-19T04:41:31,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e1601f400a734cdaa15fc21e95f82cc5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e1601f400a734cdaa15fc21e95f82cc5 2024-11-19T04:41:31,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5838c4623088426ba6d906ea708602ad to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5838c4623088426ba6d906ea708602ad 2024-11-19T04:41:31,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/bd48ffca86bc44bda6b4ceef68851c81 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/bd48ffca86bc44bda6b4ceef68851c81 2024-11-19T04:41:31,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ec758aef7b1a42da8352ab3402140ae1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ec758aef7b1a42da8352ab3402140ae1 2024-11-19T04:41:31,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/753bafeb219f43b39ebc041eca5c5233 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/753bafeb219f43b39ebc041eca5c5233 2024-11-19T04:41:31,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/47a98436c6b94415b516d7e641543649 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/47a98436c6b94415b516d7e641543649 2024-11-19T04:41:31,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4681891d78f5467a924f3576de159e58 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4681891d78f5467a924f3576de159e58 2024-11-19T04:41:31,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ffdcce9769a4e3d85865038341ff301 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ffdcce9769a4e3d85865038341ff301 2024-11-19T04:41:31,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e79a21f27107469bb34f030178cbb03f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/e79a21f27107469bb34f030178cbb03f 2024-11-19T04:41:31,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ecfc3affce945f29425bc4e07a35f9e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/9ecfc3affce945f29425bc4e07a35f9e 2024-11-19T04:41:31,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/2b661a90cf5f4735b675e8c1c07c701d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/2b661a90cf5f4735b675e8c1c07c701d 2024-11-19T04:41:31,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/6023ed92e70446798acaa4f5b83b56e3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/6023ed92e70446798acaa4f5b83b56e3 2024-11-19T04:41:31,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4ea96a592adc40b9a8ad18845feac513 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/4ea96a592adc40b9a8ad18845feac513 2024-11-19T04:41:31,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/8a4f455df4ae417a8f26e17d3d0d12f5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/8a4f455df4ae417a8f26e17d3d0d12f5 2024-11-19T04:41:31,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/aaeed791592047109d5d96aab5f70a93 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/aaeed791592047109d5d96aab5f70a93 2024-11-19T04:41:31,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5de4fa364cfe43aa9bb58f049b35b987 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/5de4fa364cfe43aa9bb58f049b35b987 2024-11-19T04:41:31,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b386836b8c7c4dfb8ba64537982942fa to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b386836b8c7c4dfb8ba64537982942fa 2024-11-19T04:41:31,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/92a77de69ab040a1acf715a554419740 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/92a77de69ab040a1acf715a554419740 2024-11-19T04:41:31,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/eb1a534010b34920b7be72a2e3847a73 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/eb1a534010b34920b7be72a2e3847a73 2024-11-19T04:41:31,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/34da0d9a411b4936b26b0dcc15d05888 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/34da0d9a411b4936b26b0dcc15d05888 2024-11-19T04:41:31,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f1ef6e1ba12d4bf0aa9d4c3ca0ca21d5 2024-11-19T04:41:31,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ff1bb33208834124b2df41f05744e2de to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/ff1bb33208834124b2df41f05744e2de 2024-11-19T04:41:31,064 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/recovered.edits/517.seqid, newMaxSeqId=517, maxSeqId=4 2024-11-19T04:41:31,065 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. 2024-11-19T04:41:31,065 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1676): Region close journal for 2579823445e39a7eaea2aba4ffb4cf67: Waiting for close lock at 1731991289310Running coprocessor pre-close hooks at 1731991289310Disabling compacts and flushes for region at 1731991289310Disabling writes for close at 1731991289310Obtaining lock to block concurrent updates at 1731991289310Preparing flush snapshotting stores in 2579823445e39a7eaea2aba4ffb4cf67 at 1731991289310Finished memstore snapshotting TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731991289311 (+1 ms)Flushing stores of TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67. at 1731991289311Flushing 2579823445e39a7eaea2aba4ffb4cf67/A: creating writer at 1731991289311Flushing mob file 2579823445e39a7eaea2aba4ffb4cf67/A: appending metadata at 1731991289316 (+5 ms)Flushing mob file 2579823445e39a7eaea2aba4ffb4cf67/A: closing flushed file at 1731991289316Flushing 2579823445e39a7eaea2aba4ffb4cf67/A: appending metadata at 1731991289724 (+408 ms)Flushing 2579823445e39a7eaea2aba4ffb4cf67/A: closing flushed file at 1731991289725 (+1 ms)Flushing 2579823445e39a7eaea2aba4ffb4cf67/B: creating writer at 1731991290133 (+408 ms)Flushing 2579823445e39a7eaea2aba4ffb4cf67/B: appending metadata at 1731991290135 (+2 ms)Flushing 2579823445e39a7eaea2aba4ffb4cf67/B: closing flushed file at 1731991290135Flushing 2579823445e39a7eaea2aba4ffb4cf67/C: creating writer at 1731991290543 (+408 ms)Flushing 2579823445e39a7eaea2aba4ffb4cf67/C: appending metadata at 1731991290545 (+2 ms)Flushing 2579823445e39a7eaea2aba4ffb4cf67/C: closing flushed file at 1731991290545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c23df57: reopening flushed file at 1731991290954 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26a4c707: reopening flushed file at 1731991290958 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a1f497e: reopening flushed file at 1731991290961 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2579823445e39a7eaea2aba4ffb4cf67 in 1656ms, sequenceid=514, compaction requested=true at 1731991290966 (+5 ms)Writing region close event to WAL at 1731991291062 (+96 ms)Running coprocessor post-close hooks at 1731991291065 (+3 ms)Closed at 1731991291065 2024-11-19T04:41:31,067 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] handler.UnassignRegionHandler(157): Closed 2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=2579823445e39a7eaea2aba4ffb4cf67, regionState=CLOSED 2024-11-19T04:41:31,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:41:31,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T04:41:31,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; CloseRegionProcedure 2579823445e39a7eaea2aba4ffb4cf67, server=08a7f35e60d4,32975,1731991233019 in 1.9120 sec 2024-11-19T04:41:31,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-19T04:41:31,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2579823445e39a7eaea2aba4ffb4cf67, UNASSIGN in 1.9160 sec 2024-11-19T04:41:31,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T04:41:31,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-19T04:41:31,076 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991291075"}]},"ts":"1731991291075"} 2024-11-19T04:41:31,077 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T04:41:31,077 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T04:41:31,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-19T04:41:31,218 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T04:41:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T04:41:31,297 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:31,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T04:41:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,300 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=68, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T04:41:31,300 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=68, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,304 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,305 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/recovered.edits] 2024-11-19T04:41:31,307 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/32c6940a3785427ab66c115e57a3de19 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/32c6940a3785427ab66c115e57a3de19 2024-11-19T04:41:31,308 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b33173a3b71242e7b02f6fa8e728bab3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/b33173a3b71242e7b02f6fa8e728bab3 2024-11-19T04:41:31,309 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/deae685a1f91449491871f579c8ba005 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/A/deae685a1f91449491871f579c8ba005 2024-11-19T04:41:31,311 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/798f62003d494b29a12374895eb22ad5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/798f62003d494b29a12374895eb22ad5 2024-11-19T04:41:31,312 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/cf1641458a964a6393526156c8cff1f4 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/cf1641458a964a6393526156c8cff1f4 2024-11-19T04:41:31,313 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d5ec1ffcb82f421f8202e56e44ae7586 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/B/d5ec1ffcb82f421f8202e56e44ae7586 2024-11-19T04:41:31,315 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b5a616dd30c6497b8381b11c4d90b736 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/b5a616dd30c6497b8381b11c4d90b736 2024-11-19T04:41:31,316 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f71ebf2961104e62ac9e7c295c6a5d1c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/C/f71ebf2961104e62ac9e7c295c6a5d1c 2024-11-19T04:41:31,318 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/recovered.edits/517.seqid to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67/recovered.edits/517.seqid 2024-11-19T04:41:31,318 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T04:41:31,319 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T04:41:31,319 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T04:41:31,323 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190f24643bc4c74ce1830c06fe967d5837_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190f24643bc4c74ce1830c06fe967d5837_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ff14c2dc0744a74bc71bceef1915469_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411190ff14c2dc0744a74bc71bceef1915469_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191b9b161ff472425989ed17d304d2d2ce_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191b9b161ff472425989ed17d304d2d2ce_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,325 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c9c85fdfc2f4af5b474b4d46d5cb97c_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191c9c85fdfc2f4af5b474b4d46d5cb97c_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,326 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111925f77981e43a4dd0b0308c7e7245b4e7_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111925f77981e43a4dd0b0308c7e7245b4e7_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,327 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f984417dd4c49b4a8becaf406cd7c5d_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192f984417dd4c49b4a8becaf406cd7c5d_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,328 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193056862a7d534fcaa794c446cff4e825_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193056862a7d534fcaa794c446cff4e825_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,329 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111933d687cb618e4bd3af4b660f558ab439_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111933d687cb618e4bd3af4b660f558ab439_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,330 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941ec737e35564dfe9c7349419de9a762_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111941ec737e35564dfe9c7349419de9a762_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,330 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119477a94640fc9465e80fd23f3cd0cc5d1_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119477a94640fc9465e80fd23f3cd0cc5d1_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,331 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119478907c15b52471e8d1ddc34bf658539_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119478907c15b52471e8d1ddc34bf658539_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,332 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be80f352f3349cd907a3ec1d622525b_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194be80f352f3349cd907a3ec1d622525b_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198053d2dc85fe427ca152f47908f9cd41_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198053d2dc85fe427ca152f47908f9cd41_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111990646900a4414b85b0e1cc64ab363eef_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111990646900a4414b85b0e1cc64ab363eef_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111993f30f58ff80482a927373927f6cabce_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111993f30f58ff80482a927373927f6cabce_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e5415de76054a51875672ba58dcb81c_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e5415de76054a51875672ba58dcb81c_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a9924dc362984ece998368efa4afd111_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a9924dc362984ece998368efa4afd111_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,339 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad984f20646545378a543487fbcabb51_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ad984f20646545378a543487fbcabb51_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,340 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b09468f7c69c48f199464a77390f27a9_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b09468f7c69c48f199464a77390f27a9_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,341 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b44fa8c28d724321b28a016168d27999_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b44fa8c28d724321b28a016168d27999_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bcf2af05c45043e4b08b400193f2fc7f_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bcf2af05c45043e4b08b400193f2fc7f_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,343 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c02a992aea9f4c00b843c951bf631c97_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c02a992aea9f4c00b843c951bf631c97_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,345 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ca875193c4b84dd8a5521ec933c5ca86_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ca875193c4b84dd8a5521ec933c5ca86_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,346 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e8854b0de3064a4f9f51460f028d5d4f_2579823445e39a7eaea2aba4ffb4cf67 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e8854b0de3064a4f9f51460f028d5d4f_2579823445e39a7eaea2aba4ffb4cf67 2024-11-19T04:41:31,346 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T04:41:31,348 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=68, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,350 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T04:41:31,352 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T04:41:31,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=68, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T04:41:31,353 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731991291353"}]},"ts":"9223372036854775807"} 2024-11-19T04:41:31,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T04:41:31,355 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2579823445e39a7eaea2aba4ffb4cf67, NAME => 'TestAcidGuarantees,,1731991263634.2579823445e39a7eaea2aba4ffb4cf67.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T04:41:31,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T04:41:31,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731991291355"}]},"ts":"9223372036854775807"} 2024-11-19T04:41:31,357 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T04:41:31,357 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=68, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-19T04:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T04:41:31,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T04:41:31,411 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:31,418 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_357640213_22 at /127.0.0.1:51142 [Waiting for operation #803] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_357640213_22 at /127.0.0.1:51132 [Waiting for operation #812] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_865084663_22 at /127.0.0.1:44598 [Waiting for operation #878] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_865084663_22 at /127.0.0.1:38810 [Waiting for operation #1246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 425) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12457 (was 12321) - AvailableMemoryMB LEAK? - 2024-11-19T04:41:31,427 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=505, ProcessCount=11, AvailableMemoryMB=12457 2024-11-19T04:41:31,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:41:31,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:41:31,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T04:41:31,430 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T04:41:31,430 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:31,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 69 2024-11-19T04:41:31,431 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T04:41:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-19T04:41:31,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742090_1266 (size=1061) 2024-11-19T04:41:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-19T04:41:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-19T04:41:31,839 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:41:31,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742091_1267 (size=53) 2024-11-19T04:41:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-19T04:41:32,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:41:32,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 641e0316dd7be3ea1908e5148ffce278, disabling compactions & flushes 2024-11-19T04:41:32,244 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,244 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. after waiting 1 ms 2024-11-19T04:41:32,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,245 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 641e0316dd7be3ea1908e5148ffce278: Waiting for close lock at 1731991292244Disabling compacts and flushes for region at 1731991292244Disabling writes for close at 1731991292245 (+1 ms)Writing region close event to WAL at 1731991292245Closed at 1731991292245 2024-11-19T04:41:32,246 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T04:41:32,246 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731991292246"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731991292246"}]},"ts":"1731991292246"} 2024-11-19T04:41:32,247 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T04:41:32,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T04:41:32,249 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991292248"}]},"ts":"1731991292248"} 2024-11-19T04:41:32,250 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T04:41:32,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, ASSIGN}] 2024-11-19T04:41:32,251 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, ASSIGN 2024-11-19T04:41:32,252 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, ASSIGN; state=OFFLINE, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=false 2024-11-19T04:41:32,403 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=641e0316dd7be3ea1908e5148ffce278, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:32,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, ASSIGN because future has completed 2024-11-19T04:41:32,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:41:32,559 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,559 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7752): Opening region: {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:41:32,559 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,560 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:41:32,560 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7794): checking encryption for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,560 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7797): checking classloading for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,561 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,562 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:32,562 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 641e0316dd7be3ea1908e5148ffce278 columnFamilyName A 2024-11-19T04:41:32,562 DEBUG [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:32,563 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.HStore(327): Store=641e0316dd7be3ea1908e5148ffce278/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:32,563 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,564 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:32,564 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 641e0316dd7be3ea1908e5148ffce278 columnFamilyName B 2024-11-19T04:41:32,564 DEBUG [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:32,564 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.HStore(327): Store=641e0316dd7be3ea1908e5148ffce278/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:32,564 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,565 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:41:32,565 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 641e0316dd7be3ea1908e5148ffce278 columnFamilyName C 2024-11-19T04:41:32,565 DEBUG [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:41:32,566 INFO [StoreOpener-641e0316dd7be3ea1908e5148ffce278-1 {}] regionserver.HStore(327): Store=641e0316dd7be3ea1908e5148ffce278/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:41:32,566 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,566 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1038): replaying wal for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,567 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,567 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-19T04:41:32,568 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1048): stopping wal replay for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,568 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1060): Cleaning up temporary data for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,568 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:41:32,569 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1093): writing seq id for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,571 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:41:32,572 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1114): Opened 641e0316dd7be3ea1908e5148ffce278; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72772933, jitterRate=0.0844012051820755}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:41:32,572 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:32,573 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1006): Region open journal for 641e0316dd7be3ea1908e5148ffce278: Running coprocessor pre-open hook at 1731991292560Writing region info on filesystem at 1731991292560Initializing all the Stores at 1731991292560Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991292560Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991292561 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991292561Cleaning up temporary data from old regions at 1731991292568 (+7 ms)Running coprocessor post-open hooks at 1731991292572 (+4 ms)Region opened successfully at 1731991292573 (+1 ms) 2024-11-19T04:41:32,574 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., pid=71, masterSystemTime=1731991292556 2024-11-19T04:41:32,576 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,576 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:32,576 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=641e0316dd7be3ea1908e5148ffce278, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:32,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:41:32,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T04:41:32,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 in 174 msec 2024-11-19T04:41:32,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-19T04:41:32,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, ASSIGN in 329 msec 2024-11-19T04:41:32,582 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T04:41:32,582 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991292582"}]},"ts":"1731991292582"} 2024-11-19T04:41:32,583 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T04:41:32,583 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T04:41:32,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-19T04:41:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-19T04:41:33,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:33,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c197e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7f17bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,583 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,585 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:41:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:41:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@932473e 2024-11-19T04:41:33,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:41:33,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:41:33,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,588 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404223e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ccdf7ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,591 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aa940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,593 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49304220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f467671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,596 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4279c18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62fda4d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,598 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af60f3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,600 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4acc52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,602 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab00e48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,603 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc22a17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,605 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764f91c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,606 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527347dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad82fab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,609 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f876bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,610 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4806bbaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,612 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276aec59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37cb29da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:41:33,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:41:33,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:41:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:41:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:41:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bd8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:41:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:41:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:33,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:41:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6ed43b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:41:33,617 DEBUG [Thread-1229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,617 DEBUG [Thread-1236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T04:41:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T04:41:33,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,619 DEBUG [Thread-1238 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,621 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:33,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,626 DEBUG [Thread-1242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,627 DEBUG [Thread-1244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,627 DEBUG [Thread-1225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,628 DEBUG [Thread-1227 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,651 DEBUG [Thread-1233 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,655 DEBUG [Thread-1240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,655 DEBUG [Thread-1231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:41:33,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,656 DEBUG [Thread-1240 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,657 DEBUG [Thread-1240 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:41:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:41:33,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:41:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:33,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:41:33,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:41:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T04:41:33,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/48ee7fc311e64483a41de90b3bd587d1 is 50, key is test_row_0/A:col10/1731991293661/Put/seqid=0 2024-11-19T04:41:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742092_1268 (size=12001) 2024-11-19T04:41:33,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/48ee7fc311e64483a41de90b3bd587d1 2024-11-19T04:41:33,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:33,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:33,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:33,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5c39bb40b25a476f8f103a0c6a9e2525 is 50, key is test_row_0/B:col10/1731991293661/Put/seqid=0 2024-11-19T04:41:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991353777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991353806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991353806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991353815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991353815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742093_1269 (size=12001) 2024-11-19T04:41:33,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991353917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:33,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:33,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:33,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:33,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:33,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:33,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991353927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991353928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991353928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991353928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T04:41:34,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:34,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:34,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991354127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991354138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991354138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991354138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991354138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T04:41:34,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5c39bb40b25a476f8f103a0c6a9e2525 2024-11-19T04:41:34,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/330f74150b84412783f5a40240134275 is 50, key is test_row_0/C:col10/1731991293661/Put/seqid=0 2024-11-19T04:41:34,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742094_1270 (size=12001) 2024-11-19T04:41:34,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/330f74150b84412783f5a40240134275 2024-11-19T04:41:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/48ee7fc311e64483a41de90b3bd587d1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/48ee7fc311e64483a41de90b3bd587d1 2024-11-19T04:41:34,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/48ee7fc311e64483a41de90b3bd587d1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:41:34,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5c39bb40b25a476f8f103a0c6a9e2525 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5c39bb40b25a476f8f103a0c6a9e2525 2024-11-19T04:41:34,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5c39bb40b25a476f8f103a0c6a9e2525, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:41:34,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/330f74150b84412783f5a40240134275 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/330f74150b84412783f5a40240134275 2024-11-19T04:41:34,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/330f74150b84412783f5a40240134275, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:41:34,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 641e0316dd7be3ea1908e5148ffce278 in 730ms, sequenceid=15, compaction requested=false 2024-11-19T04:41:34,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:34,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:41:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:34,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:34,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/4521691ef35b4df08b56e259c1dbdd51 is 50, key is test_row_0/A:col10/1731991293806/Put/seqid=0 2024-11-19T04:41:34,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991354466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991354465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991354467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991354468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991354468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742095_1271 (size=12001) 2024-11-19T04:41:34,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/4521691ef35b4df08b56e259c1dbdd51 2024-11-19T04:41:34,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5e2b5b5a16a748d39b49622e9bcbe848 is 50, key is test_row_0/B:col10/1731991293806/Put/seqid=0 2024-11-19T04:41:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742096_1272 (size=12001) 2024-11-19T04:41:34,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991354577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991354577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991354577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991354577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991354577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T04:41:34,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991354788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:34,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:34,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:34,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:34,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5e2b5b5a16a748d39b49622e9bcbe848 2024-11-19T04:41:34,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/b997f9c92a1048e0b64a5906dcb2b01c is 50, key is test_row_0/C:col10/1731991293806/Put/seqid=0 2024-11-19T04:41:34,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742097_1273 (size=12001) 2024-11-19T04:41:34,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/b997f9c92a1048e0b64a5906dcb2b01c 2024-11-19T04:41:34,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/4521691ef35b4df08b56e259c1dbdd51 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4521691ef35b4df08b56e259c1dbdd51 2024-11-19T04:41:34,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4521691ef35b4df08b56e259c1dbdd51, entries=150, sequenceid=35, filesize=11.7 K 2024-11-19T04:41:34,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5e2b5b5a16a748d39b49622e9bcbe848 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5e2b5b5a16a748d39b49622e9bcbe848 2024-11-19T04:41:34,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5e2b5b5a16a748d39b49622e9bcbe848, entries=150, sequenceid=35, filesize=11.7 K 2024-11-19T04:41:34,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/b997f9c92a1048e0b64a5906dcb2b01c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/b997f9c92a1048e0b64a5906dcb2b01c 2024-11-19T04:41:34,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/b997f9c92a1048e0b64a5906dcb2b01c, entries=150, sequenceid=35, filesize=11.7 K 2024-11-19T04:41:34,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 641e0316dd7be3ea1908e5148ffce278 in 514ms, sequenceid=35, compaction requested=false 2024-11-19T04:41:34,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:35,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:35,010 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:35,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:35,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3a07e1a1b65a468c8c315f9330a5c0f8 is 50, key is test_row_0/A:col10/1731991294455/Put/seqid=0 2024-11-19T04:41:35,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742098_1274 (size=12001) 2024-11-19T04:41:35,039 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3a07e1a1b65a468c8c315f9330a5c0f8 2024-11-19T04:41:35,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/099aa420889f46b2ac0b9d80c4fc7d71 is 50, key is test_row_0/B:col10/1731991294455/Put/seqid=0 2024-11-19T04:41:35,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991355127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991355128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742099_1275 (size=12001) 2024-11-19T04:41:35,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991355135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991355136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991355137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991355247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991355247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991355258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991355258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991355258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991355457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991355457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991355467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991355468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991355468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,543 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/099aa420889f46b2ac0b9d80c4fc7d71 2024-11-19T04:41:35,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/98c1831bba9a4a89a92376508e32012e is 50, key is test_row_0/C:col10/1731991294455/Put/seqid=0 2024-11-19T04:41:35,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742100_1276 (size=12001) 2024-11-19T04:41:35,610 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/98c1831bba9a4a89a92376508e32012e 2024-11-19T04:41:35,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3a07e1a1b65a468c8c315f9330a5c0f8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3a07e1a1b65a468c8c315f9330a5c0f8 2024-11-19T04:41:35,621 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3a07e1a1b65a468c8c315f9330a5c0f8, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T04:41:35,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/099aa420889f46b2ac0b9d80c4fc7d71 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/099aa420889f46b2ac0b9d80c4fc7d71 2024-11-19T04:41:35,627 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/099aa420889f46b2ac0b9d80c4fc7d71, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T04:41:35,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/98c1831bba9a4a89a92376508e32012e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/98c1831bba9a4a89a92376508e32012e 2024-11-19T04:41:35,633 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/98c1831bba9a4a89a92376508e32012e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-19T04:41:35,633 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 641e0316dd7be3ea1908e5148ffce278 in 623ms, sequenceid=49, compaction requested=true 2024-11-19T04:41:35,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:35,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:35,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T04:41:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T04:41:35,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T04:41:35,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-11-19T04:41:35,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0240 sec 2024-11-19T04:41:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T04:41:35,769 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:35,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:35,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-11-19T04:41:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:35,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:35,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:35,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:35,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:35,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:35,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T04:41:35,776 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:35,777 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T04:41:35,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:35,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/4cf2a87810a64c28895981d68f5678b0 is 50, key is test_row_0/A:col10/1731991295771/Put/seqid=0 2024-11-19T04:41:35,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991355785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991355791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991355793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991355793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991355795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742101_1277 (size=16681) 2024-11-19T04:41:35,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/4cf2a87810a64c28895981d68f5678b0 2024-11-19T04:41:35,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/d98ac311fb6143e490729e8d33e1dc14 is 50, key is test_row_0/B:col10/1731991295771/Put/seqid=0 2024-11-19T04:41:35,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742102_1278 (size=12001) 2024-11-19T04:41:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T04:41:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991355908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991355908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991355908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991355908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991355908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:35,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T04:41:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:35,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T04:41:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T04:41:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991356116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991356117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991356117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991356117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991356117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/d98ac311fb6143e490729e8d33e1dc14 2024-11-19T04:41:36,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T04:41:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:36,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/20252e11b1a148feaf42d365f8846e19 is 50, key is test_row_0/C:col10/1731991295771/Put/seqid=0 2024-11-19T04:41:36,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742103_1279 (size=12001) 2024-11-19T04:41:36,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/20252e11b1a148feaf42d365f8846e19 2024-11-19T04:41:36,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/4cf2a87810a64c28895981d68f5678b0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4cf2a87810a64c28895981d68f5678b0 2024-11-19T04:41:36,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4cf2a87810a64c28895981d68f5678b0, entries=250, sequenceid=70, filesize=16.3 K 2024-11-19T04:41:36,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/d98ac311fb6143e490729e8d33e1dc14 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/d98ac311fb6143e490729e8d33e1dc14 2024-11-19T04:41:36,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/d98ac311fb6143e490729e8d33e1dc14, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T04:41:36,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/20252e11b1a148feaf42d365f8846e19 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/20252e11b1a148feaf42d365f8846e19 2024-11-19T04:41:36,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/20252e11b1a148feaf42d365f8846e19, entries=150, sequenceid=70, filesize=11.7 K 2024-11-19T04:41:36,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 641e0316dd7be3ea1908e5148ffce278 in 545ms, sequenceid=70, compaction requested=true 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:36,318 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:36,318 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:36,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:36,320 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:36,320 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:36,320 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,321 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/48ee7fc311e64483a41de90b3bd587d1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4521691ef35b4df08b56e259c1dbdd51, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3a07e1a1b65a468c8c315f9330a5c0f8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4cf2a87810a64c28895981d68f5678b0] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=51.4 K 2024-11-19T04:41:36,321 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:36,321 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:36,321 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,321 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5c39bb40b25a476f8f103a0c6a9e2525, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5e2b5b5a16a748d39b49622e9bcbe848, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/099aa420889f46b2ac0b9d80c4fc7d71, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/d98ac311fb6143e490729e8d33e1dc14] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=46.9 K 2024-11-19T04:41:36,322 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48ee7fc311e64483a41de90b3bd587d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991293661 2024-11-19T04:41:36,322 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c39bb40b25a476f8f103a0c6a9e2525, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991293661 2024-11-19T04:41:36,322 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4521691ef35b4df08b56e259c1dbdd51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731991293777 2024-11-19T04:41:36,322 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2b5b5a16a748d39b49622e9bcbe848, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731991293777 2024-11-19T04:41:36,322 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a07e1a1b65a468c8c315f9330a5c0f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731991294455 2024-11-19T04:41:36,323 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 099aa420889f46b2ac0b9d80c4fc7d71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731991294455 2024-11-19T04:41:36,324 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d98ac311fb6143e490729e8d33e1dc14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731991295135 2024-11-19T04:41:36,324 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cf2a87810a64c28895981d68f5678b0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731991295125 2024-11-19T04:41:36,347 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:36,348 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/f8a67afeb93c4dfcb6603519dfcf0263 is 50, key is test_row_0/B:col10/1731991295771/Put/seqid=0 2024-11-19T04:41:36,360 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:36,360 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/6bbf11afda84450a9e12a75f8e671d29 is 50, key is test_row_0/A:col10/1731991295771/Put/seqid=0 2024-11-19T04:41:36,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742104_1280 (size=12139) 2024-11-19T04:41:36,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742105_1281 (size=12139) 2024-11-19T04:41:36,386 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/f8a67afeb93c4dfcb6603519dfcf0263 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f8a67afeb93c4dfcb6603519dfcf0263 2024-11-19T04:41:36,391 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into f8a67afeb93c4dfcb6603519dfcf0263(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:36,391 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:36,391 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=12, startTime=1731991296318; duration=0sec 2024-11-19T04:41:36,392 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:36,392 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:36,392 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:36,393 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:36,393 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:36,394 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,394 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/330f74150b84412783f5a40240134275, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/b997f9c92a1048e0b64a5906dcb2b01c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/98c1831bba9a4a89a92376508e32012e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/20252e11b1a148feaf42d365f8846e19] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=46.9 K 2024-11-19T04:41:36,394 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 330f74150b84412783f5a40240134275, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991293661 2024-11-19T04:41:36,395 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b997f9c92a1048e0b64a5906dcb2b01c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1731991293777 2024-11-19T04:41:36,395 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c1831bba9a4a89a92376508e32012e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731991294455 2024-11-19T04:41:36,395 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 20252e11b1a148feaf42d365f8846e19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731991295135 2024-11-19T04:41:36,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T04:41:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,404 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:41:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:36,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T04:41:36,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3f4b305c665c41cda5b2606a9f9f032a is 50, key is test_row_0/A:col10/1731991295793/Put/seqid=0 2024-11-19T04:41:36,428 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:36,428 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/718476832b614784872fb97b31580c18 is 50, key is test_row_0/C:col10/1731991295771/Put/seqid=0 2024-11-19T04:41:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:36,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:36,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742106_1282 (size=12001) 2024-11-19T04:41:36,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742107_1283 (size=12139) 2024-11-19T04:41:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991356453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991356456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991356457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731991296450, max timestamp=1731991296457Num uniques 859; 2024-11-19T04:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731991296450, max timestamp=1731991296457Num uniques 859; 2024-11-19T04:41:36,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:36,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731991296450, max timestamp=1731991296457Num uniques 859; 2024-11-19T04:41:36,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:36,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991356505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991356511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991356567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991356567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991356567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,591 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T04:41:36,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991356617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991356627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991356777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991356777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991356777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,796 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/6bbf11afda84450a9e12a75f8e671d29 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6bbf11afda84450a9e12a75f8e671d29 2024-11-19T04:41:36,802 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 6bbf11afda84450a9e12a75f8e671d29(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:36,802 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:36,802 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=12, startTime=1731991296318; duration=0sec 2024-11-19T04:41:36,802 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:36,802 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991356827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,840 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3f4b305c665c41cda5b2606a9f9f032a 2024-11-19T04:41:36,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991356837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:36,850 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/718476832b614784872fb97b31580c18 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/718476832b614784872fb97b31580c18 2024-11-19T04:41:36,857 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into 718476832b614784872fb97b31580c18(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:36,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:36,857 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=12, startTime=1731991296318; duration=0sec 2024-11-19T04:41:36,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:36,857 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:36,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e3f7182c4a7c4a1d9410b09823cc5e7f is 50, key is test_row_0/B:col10/1731991295793/Put/seqid=0 2024-11-19T04:41:36,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742108_1284 (size=12001) 2024-11-19T04:41:36,875 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e3f7182c4a7c4a1d9410b09823cc5e7f 2024-11-19T04:41:36,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/6894afe7829b4f80b248e8f1c481d4a7 is 50, key is test_row_0/C:col10/1731991295793/Put/seqid=0 2024-11-19T04:41:36,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742109_1285 (size=12001) 2024-11-19T04:41:36,916 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/6894afe7829b4f80b248e8f1c481d4a7 2024-11-19T04:41:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T04:41:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3f4b305c665c41cda5b2606a9f9f032a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3f4b305c665c41cda5b2606a9f9f032a 2024-11-19T04:41:36,929 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3f4b305c665c41cda5b2606a9f9f032a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-19T04:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e3f7182c4a7c4a1d9410b09823cc5e7f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e3f7182c4a7c4a1d9410b09823cc5e7f 2024-11-19T04:41:36,935 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e3f7182c4a7c4a1d9410b09823cc5e7f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-19T04:41:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/6894afe7829b4f80b248e8f1c481d4a7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/6894afe7829b4f80b248e8f1c481d4a7 2024-11-19T04:41:36,942 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/6894afe7829b4f80b248e8f1c481d4a7, entries=150, sequenceid=82, filesize=11.7 K 2024-11-19T04:41:36,943 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 641e0316dd7be3ea1908e5148ffce278 in 540ms, sequenceid=82, compaction requested=false 2024-11-19T04:41:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:36,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T04:41:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T04:41:36,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T04:41:36,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1680 sec 2024-11-19T04:41:36,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.1810 sec 2024-11-19T04:41:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:37,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-19T04:41:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:37,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:37,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/ebf24646edba4cc990df3d370a4a46ce is 50, key is test_row_0/A:col10/1731991296495/Put/seqid=0 2024-11-19T04:41:37,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991357108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991357109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991357109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742110_1286 (size=16681) 2024-11-19T04:41:37,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/ebf24646edba4cc990df3d370a4a46ce 2024-11-19T04:41:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991357137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e791a8b5f96d4ecbab1fa435e1632f58 is 50, key is test_row_0/B:col10/1731991296495/Put/seqid=0 2024-11-19T04:41:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991357147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742111_1287 (size=12001) 2024-11-19T04:41:37,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e791a8b5f96d4ecbab1fa435e1632f58 2024-11-19T04:41:37,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a5f686b98a6646f5b13ca2989eeee919 is 50, key is test_row_0/C:col10/1731991296495/Put/seqid=0 2024-11-19T04:41:37,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742112_1288 (size=12001) 2024-11-19T04:41:37,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991357218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991357227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991357227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991357427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991357437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991357437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a5f686b98a6646f5b13ca2989eeee919 2024-11-19T04:41:37,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/ebf24646edba4cc990df3d370a4a46ce as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ebf24646edba4cc990df3d370a4a46ce 2024-11-19T04:41:37,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ebf24646edba4cc990df3d370a4a46ce, entries=250, sequenceid=121, filesize=16.3 K 2024-11-19T04:41:37,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e791a8b5f96d4ecbab1fa435e1632f58 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e791a8b5f96d4ecbab1fa435e1632f58 2024-11-19T04:41:37,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e791a8b5f96d4ecbab1fa435e1632f58, entries=150, sequenceid=121, filesize=11.7 K 2024-11-19T04:41:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a5f686b98a6646f5b13ca2989eeee919 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a5f686b98a6646f5b13ca2989eeee919 2024-11-19T04:41:37,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a5f686b98a6646f5b13ca2989eeee919, entries=150, sequenceid=121, filesize=11.7 K 2024-11-19T04:41:37,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 641e0316dd7be3ea1908e5148ffce278 in 539ms, sequenceid=121, compaction requested=true 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:37,631 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:37,631 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:37,632 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:37,632 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:37,632 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:37,633 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:37,633 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:37,633 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:37,633 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6bbf11afda84450a9e12a75f8e671d29, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3f4b305c665c41cda5b2606a9f9f032a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ebf24646edba4cc990df3d370a4a46ce] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=39.9 K 2024-11-19T04:41:37,633 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f8a67afeb93c4dfcb6603519dfcf0263, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e3f7182c4a7c4a1d9410b09823cc5e7f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e791a8b5f96d4ecbab1fa435e1632f58] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=35.3 K 2024-11-19T04:41:37,635 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a67afeb93c4dfcb6603519dfcf0263, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731991295135 2024-11-19T04:41:37,635 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bbf11afda84450a9e12a75f8e671d29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731991295135 2024-11-19T04:41:37,638 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f4b305c665c41cda5b2606a9f9f032a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731991295783 2024-11-19T04:41:37,639 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f7182c4a7c4a1d9410b09823cc5e7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731991295783 2024-11-19T04:41:37,639 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf24646edba4cc990df3d370a4a46ce, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731991296495 2024-11-19T04:41:37,639 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e791a8b5f96d4ecbab1fa435e1632f58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731991296495 2024-11-19T04:41:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:37,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:41:37,650 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:37,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:37,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:37,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:37,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:37,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:37,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:37,651 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/f29c082c88a447329a64db85429eb719 is 50, key is test_row_0/A:col10/1731991296495/Put/seqid=0 2024-11-19T04:41:37,666 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:37,667 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/c356e9a471734213a6868e81fa462e38 is 50, key is test_row_0/B:col10/1731991296495/Put/seqid=0 2024-11-19T04:41:37,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/ed5a8ed8bc8b45c488db073cd49557aa is 50, key is test_row_0/A:col10/1731991297648/Put/seqid=0 2024-11-19T04:41:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742113_1289 (size=12241) 2024-11-19T04:41:37,706 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/f29c082c88a447329a64db85429eb719 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f29c082c88a447329a64db85429eb719 2024-11-19T04:41:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742114_1290 (size=12241) 2024-11-19T04:41:37,714 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into f29c082c88a447329a64db85429eb719(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:37,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:37,714 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991297631; duration=0sec 2024-11-19T04:41:37,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:37,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:37,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:37,719 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:37,719 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:37,719 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:37,719 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/718476832b614784872fb97b31580c18, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/6894afe7829b4f80b248e8f1c481d4a7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a5f686b98a6646f5b13ca2989eeee919] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=35.3 K 2024-11-19T04:41:37,721 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 718476832b614784872fb97b31580c18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731991295135 2024-11-19T04:41:37,721 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6894afe7829b4f80b248e8f1c481d4a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731991295783 2024-11-19T04:41:37,722 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f686b98a6646f5b13ca2989eeee919, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731991296495 2024-11-19T04:41:37,723 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/c356e9a471734213a6868e81fa462e38 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c356e9a471734213a6868e81fa462e38 2024-11-19T04:41:37,729 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into c356e9a471734213a6868e81fa462e38(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:37,729 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:37,729 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991297631; duration=0sec 2024-11-19T04:41:37,729 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:37,730 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:37,734 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:37,735 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/37483eb4a4bb4aca9a09f3a4680e2882 is 50, key is test_row_0/C:col10/1731991296495/Put/seqid=0 2024-11-19T04:41:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742115_1291 (size=14441) 2024-11-19T04:41:37,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991297704, max timestamp=1731991297743Num uniques 859; 2024-11-19T04:41:37,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:37,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991297704, max timestamp=1731991297743Num uniques 859; 2024-11-19T04:41:37,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991297704, max timestamp=1731991297743Num uniques 859; 2024-11-19T04:41:37,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:37,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:37,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742116_1292 (size=12241) 2024-11-19T04:41:37,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991357769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991357770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991357777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991357778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991357781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991357887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991357887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991357887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991357897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991357897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T04:41:37,927 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:37,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T04:41:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T04:41:37,931 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:37,932 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:37,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T04:41:38,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T04:41:38,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:38,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991358097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991358097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991358097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991358107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991358107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/ed5a8ed8bc8b45c488db073cd49557aa 2024-11-19T04:41:38,159 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/37483eb4a4bb4aca9a09f3a4680e2882 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/37483eb4a4bb4aca9a09f3a4680e2882 2024-11-19T04:41:38,166 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into 37483eb4a4bb4aca9a09f3a4680e2882(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:38,166 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:38,166 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991297631; duration=0sec 2024-11-19T04:41:38,166 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:38,166 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:38,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e0e36442357c4993a006885e0daf71b0 is 50, key is test_row_0/B:col10/1731991297648/Put/seqid=0 2024-11-19T04:41:38,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742117_1293 (size=12051) 2024-11-19T04:41:38,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e0e36442357c4993a006885e0daf71b0 2024-11-19T04:41:38,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/490a0259774348f883892e3c00e0b470 is 50, key is test_row_0/C:col10/1731991297648/Put/seqid=0 2024-11-19T04:41:38,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742118_1294 (size=12051) 2024-11-19T04:41:38,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T04:41:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T04:41:38,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T04:41:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991358407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991358407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991358408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991358417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991358417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T04:41:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T04:41:38,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/490a0259774348f883892e3c00e0b470 2024-11-19T04:41:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/ed5a8ed8bc8b45c488db073cd49557aa as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ed5a8ed8bc8b45c488db073cd49557aa 2024-11-19T04:41:38,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ed5a8ed8bc8b45c488db073cd49557aa, entries=200, sequenceid=131, filesize=14.1 K 2024-11-19T04:41:38,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e0e36442357c4993a006885e0daf71b0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e0e36442357c4993a006885e0daf71b0 2024-11-19T04:41:38,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e0e36442357c4993a006885e0daf71b0, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T04:41:38,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/490a0259774348f883892e3c00e0b470 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/490a0259774348f883892e3c00e0b470 2024-11-19T04:41:38,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/490a0259774348f883892e3c00e0b470, entries=150, sequenceid=131, filesize=11.8 K 2024-11-19T04:41:38,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 641e0316dd7be3ea1908e5148ffce278 in 1014ms, sequenceid=131, compaction requested=false 2024-11-19T04:41:38,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:38,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T04:41:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:38,702 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T04:41:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:38,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/9e9c055fb5ee424dbb7e024020c6b162 is 50, key is test_row_0/A:col10/1731991297768/Put/seqid=0 2024-11-19T04:41:38,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742119_1295 (size=12151) 2024-11-19T04:41:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:38,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:38,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991358930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991358934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991358934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991358935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991358935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991359047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991359047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991359047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991359047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991359047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T04:41:39,123 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/9e9c055fb5ee424dbb7e024020c6b162 2024-11-19T04:41:39,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/ae25f318e552400e84d71cdc38c94ecc is 50, key is test_row_0/B:col10/1731991297768/Put/seqid=0 2024-11-19T04:41:39,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742120_1296 (size=12151) 2024-11-19T04:41:39,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991359257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991359257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991359257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991359257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991359257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,537 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/ae25f318e552400e84d71cdc38c94ecc 2024-11-19T04:41:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/96a5534fe29a48ed805640843f24bd93 is 50, key is test_row_0/C:col10/1731991297768/Put/seqid=0 2024-11-19T04:41:39,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742121_1297 (size=12151) 2024-11-19T04:41:39,555 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/96a5534fe29a48ed805640843f24bd93 2024-11-19T04:41:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/9e9c055fb5ee424dbb7e024020c6b162 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9e9c055fb5ee424dbb7e024020c6b162 2024-11-19T04:41:39,571 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9e9c055fb5ee424dbb7e024020c6b162, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T04:41:39,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/ae25f318e552400e84d71cdc38c94ecc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/ae25f318e552400e84d71cdc38c94ecc 2024-11-19T04:41:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991359567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991359567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991359567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991359568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991359568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:39,581 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/ae25f318e552400e84d71cdc38c94ecc, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T04:41:39,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/96a5534fe29a48ed805640843f24bd93 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/96a5534fe29a48ed805640843f24bd93 2024-11-19T04:41:39,589 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/96a5534fe29a48ed805640843f24bd93, entries=150, sequenceid=171, filesize=11.9 K 2024-11-19T04:41:39,591 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=60.38 KB/61830 for 641e0316dd7be3ea1908e5148ffce278 in 889ms, sequenceid=171, compaction requested=true 2024-11-19T04:41:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:39,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T04:41:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T04:41:39,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T04:41:39,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6610 sec 2024-11-19T04:41:39,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6680 sec 2024-11-19T04:41:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T04:41:40,077 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:40,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-19T04:41:40,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:41:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:40,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:40,085 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T04:41:40,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:40,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:40,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/59626a72153348b5a21ea730de0ca076 is 50, key is test_row_0/A:col10/1731991300078/Put/seqid=0 2024-11-19T04:41:40,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742122_1298 (size=14541) 2024-11-19T04:41:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991360107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991360110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991360114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991360115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991360116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T04:41:40,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991360226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991360227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991360227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991360227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991360227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T04:41:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991360437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991360437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991360437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991360437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991360437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/59626a72153348b5a21ea730de0ca076 2024-11-19T04:41:40,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/934f8c97126e4d7cbadb605ed3e8e402 is 50, key is test_row_0/B:col10/1731991300078/Put/seqid=0 2024-11-19T04:41:40,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742123_1299 (size=12151) 2024-11-19T04:41:40,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T04:41:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991360747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991360748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991360748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991360748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991360757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:40,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:40,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:40,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/934f8c97126e4d7cbadb605ed3e8e402 2024-11-19T04:41:40,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/e5f1141754ec4089a1761f805ef9f412 is 50, key is test_row_0/C:col10/1731991300078/Put/seqid=0 2024-11-19T04:41:40,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742124_1300 (size=12151) 2024-11-19T04:41:41,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:41,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T04:41:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991361257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991361257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991361267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991361267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:41,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991361267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:41,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:41,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/e5f1141754ec4089a1761f805ef9f412 2024-11-19T04:41:41,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/59626a72153348b5a21ea730de0ca076 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/59626a72153348b5a21ea730de0ca076 2024-11-19T04:41:41,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/59626a72153348b5a21ea730de0ca076, entries=200, sequenceid=184, filesize=14.2 K 2024-11-19T04:41:41,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/934f8c97126e4d7cbadb605ed3e8e402 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/934f8c97126e4d7cbadb605ed3e8e402 2024-11-19T04:41:41,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/934f8c97126e4d7cbadb605ed3e8e402, entries=150, sequenceid=184, filesize=11.9 K 2024-11-19T04:41:41,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/e5f1141754ec4089a1761f805ef9f412 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/e5f1141754ec4089a1761f805ef9f412 2024-11-19T04:41:41,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/e5f1141754ec4089a1761f805ef9f412, entries=150, sequenceid=184, filesize=11.9 K 2024-11-19T04:41:41,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 641e0316dd7be3ea1908e5148ffce278 in 1314ms, sequenceid=184, compaction requested=true 2024-11-19T04:41:41,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:41,395 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:41,396 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:41,396 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:41,397 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:41,397 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f29c082c88a447329a64db85429eb719, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ed5a8ed8bc8b45c488db073cd49557aa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9e9c055fb5ee424dbb7e024020c6b162, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/59626a72153348b5a21ea730de0ca076] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=52.1 K 2024-11-19T04:41:41,397 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:41,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f29c082c88a447329a64db85429eb719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731991296495 2024-11-19T04:41:41,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5a8ed8bc8b45c488db073cd49557aa, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731991297098 2024-11-19T04:41:41,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e9c055fb5ee424dbb7e024020c6b162, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731991297767 2024-11-19T04:41:41,399 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59626a72153348b5a21ea730de0ca076, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731991298919 2024-11-19T04:41:41,399 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:41,399 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:41,399 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,399 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c356e9a471734213a6868e81fa462e38, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e0e36442357c4993a006885e0daf71b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/ae25f318e552400e84d71cdc38c94ecc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/934f8c97126e4d7cbadb605ed3e8e402] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=47.5 K 2024-11-19T04:41:41,399 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c356e9a471734213a6868e81fa462e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731991296495 2024-11-19T04:41:41,400 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e36442357c4993a006885e0daf71b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731991297098 2024-11-19T04:41:41,400 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ae25f318e552400e84d71cdc38c94ecc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731991297767 2024-11-19T04:41:41,401 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 934f8c97126e4d7cbadb605ed3e8e402, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731991298919 2024-11-19T04:41:41,412 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#249 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:41,412 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/478d27bdd3414e26832a1a88fb06ce00 is 50, key is test_row_0/A:col10/1731991300078/Put/seqid=0 2024-11-19T04:41:41,421 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:41,422 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/0600d81834964b749e403a8ef2fc673c is 50, key is test_row_0/B:col10/1731991300078/Put/seqid=0 2024-11-19T04:41:41,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742125_1301 (size=12527) 2024-11-19T04:41:41,434 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/478d27bdd3414e26832a1a88fb06ce00 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/478d27bdd3414e26832a1a88fb06ce00 2024-11-19T04:41:41,439 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 478d27bdd3414e26832a1a88fb06ce00(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:41,439 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:41,439 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=12, startTime=1731991301395; duration=0sec 2024-11-19T04:41:41,440 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:41,440 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:41,440 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:41:41,442 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:41:41,442 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:41,442 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,442 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/37483eb4a4bb4aca9a09f3a4680e2882, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/490a0259774348f883892e3c00e0b470, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/96a5534fe29a48ed805640843f24bd93, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/e5f1141754ec4089a1761f805ef9f412] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=47.5 K 2024-11-19T04:41:41,442 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37483eb4a4bb4aca9a09f3a4680e2882, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731991296495 2024-11-19T04:41:41,443 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 490a0259774348f883892e3c00e0b470, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731991297098 2024-11-19T04:41:41,443 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a5534fe29a48ed805640843f24bd93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731991297767 2024-11-19T04:41:41,444 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f1141754ec4089a1761f805ef9f412, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731991298919 2024-11-19T04:41:41,459 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:41,460 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/26b4fba33da4480bb8a2e547b1d1d0a5 is 50, key is test_row_0/C:col10/1731991300078/Put/seqid=0 2024-11-19T04:41:41,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742126_1302 (size=12527) 2024-11-19T04:41:41,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-19T04:41:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:41,467 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T04:41:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:41,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/90b0dc91ea774bcba3a75a72963cb057 is 50, key is test_row_0/A:col10/1731991300114/Put/seqid=0 2024-11-19T04:41:41,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742127_1303 (size=12527) 2024-11-19T04:41:41,502 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/26b4fba33da4480bb8a2e547b1d1d0a5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/26b4fba33da4480bb8a2e547b1d1d0a5 2024-11-19T04:41:41,509 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into 26b4fba33da4480bb8a2e547b1d1d0a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:41,509 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:41,510 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=12, startTime=1731991301397; duration=0sec 2024-11-19T04:41:41,510 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:41,510 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:41,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742128_1304 (size=12151) 2024-11-19T04:41:41,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/0600d81834964b749e403a8ef2fc673c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0600d81834964b749e403a8ef2fc673c 2024-11-19T04:41:41,889 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 0600d81834964b749e403a8ef2fc673c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:41,891 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:41,891 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=12, startTime=1731991301397; duration=0sec 2024-11-19T04:41:41,891 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:41,891 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:41,928 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/90b0dc91ea774bcba3a75a72963cb057 2024-11-19T04:41:41,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/118242453ade48a5bc0504593ff17023 is 50, key is test_row_0/B:col10/1731991300114/Put/seqid=0 2024-11-19T04:41:41,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742129_1305 (size=12151) 2024-11-19T04:41:41,982 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/118242453ade48a5bc0504593ff17023 2024-11-19T04:41:42,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ceffa79a7cca40ef82967a0126141507 is 50, key is test_row_0/C:col10/1731991300114/Put/seqid=0 2024-11-19T04:41:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742130_1306 (size=12151) 2024-11-19T04:41:42,045 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ceffa79a7cca40ef82967a0126141507 2024-11-19T04:41:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/90b0dc91ea774bcba3a75a72963cb057 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/90b0dc91ea774bcba3a75a72963cb057 2024-11-19T04:41:42,058 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/90b0dc91ea774bcba3a75a72963cb057, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T04:41:42,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/118242453ade48a5bc0504593ff17023 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/118242453ade48a5bc0504593ff17023 2024-11-19T04:41:42,067 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/118242453ade48a5bc0504593ff17023, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T04:41:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ceffa79a7cca40ef82967a0126141507 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ceffa79a7cca40ef82967a0126141507 2024-11-19T04:41:42,073 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ceffa79a7cca40ef82967a0126141507, entries=150, sequenceid=204, filesize=11.9 K 2024-11-19T04:41:42,074 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 641e0316dd7be3ea1908e5148ffce278 in 607ms, sequenceid=204, compaction requested=false 2024-11-19T04:41:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-19T04:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-19T04:41:42,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T04:41:42,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9890 sec 2024-11-19T04:41:42,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.0010 sec 2024-11-19T04:41:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T04:41:42,237 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:42,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-19T04:41:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:42,241 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:42,242 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:42,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:42,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:42,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:42,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/29fdfaf021b74e34a0df233ee86aceec is 50, key is test_row_0/A:col10/1731991302282/Put/seqid=0 2024-11-19T04:41:42,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742131_1307 (size=12151) 2024-11-19T04:41:42,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/29fdfaf021b74e34a0df233ee86aceec 2024-11-19T04:41:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991302300, max timestamp=1731991302308Num uniques 859; 2024-11-19T04:41:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991302300, max timestamp=1731991302308Num uniques 859; 2024-11-19T04:41:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991302300, max timestamp=1731991302308Num uniques 859; 2024-11-19T04:41:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:42,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:42,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/c3d7654f5d3642b1806bdd6149659ac4 is 50, key is test_row_0/B:col10/1731991302282/Put/seqid=0 2024-11-19T04:41:42,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742132_1308 (size=12151) 2024-11-19T04:41:42,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991362335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991362337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991362342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991362346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991362347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991362457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991362457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991362457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991362468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991362468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:42,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:42,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991362667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991362667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991362667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991362677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991362677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/c3d7654f5d3642b1806bdd6149659ac4 2024-11-19T04:41:42,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/1b7de0a7721d4e00a03a2c78d57a1547 is 50, key is test_row_0/C:col10/1731991302282/Put/seqid=0 2024-11-19T04:41:42,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742133_1309 (size=12151) 2024-11-19T04:41:42,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:42,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991362977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991362977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991362978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991362986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991362986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:42,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:43,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/1b7de0a7721d4e00a03a2c78d57a1547 2024-11-19T04:41:43,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/29fdfaf021b74e34a0df233ee86aceec as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/29fdfaf021b74e34a0df233ee86aceec 2024-11-19T04:41:43,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/29fdfaf021b74e34a0df233ee86aceec, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T04:41:43,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/c3d7654f5d3642b1806bdd6149659ac4 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c3d7654f5d3642b1806bdd6149659ac4 2024-11-19T04:41:43,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c3d7654f5d3642b1806bdd6149659ac4, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T04:41:43,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/1b7de0a7721d4e00a03a2c78d57a1547 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1b7de0a7721d4e00a03a2c78d57a1547 2024-11-19T04:41:43,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:43,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:43,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1b7de0a7721d4e00a03a2c78d57a1547, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T04:41:43,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 641e0316dd7be3ea1908e5148ffce278 in 885ms, sequenceid=216, compaction requested=true 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:43,169 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:43,169 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:43,170 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:43,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:43,170 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:43,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:43,170 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,170 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0600d81834964b749e403a8ef2fc673c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/118242453ade48a5bc0504593ff17023, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c3d7654f5d3642b1806bdd6149659ac4] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.0 K 2024-11-19T04:41:43,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/478d27bdd3414e26832a1a88fb06ce00, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/90b0dc91ea774bcba3a75a72963cb057, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/29fdfaf021b74e34a0df233ee86aceec] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.0 K 2024-11-19T04:41:43,171 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0600d81834964b749e403a8ef2fc673c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731991298919 2024-11-19T04:41:43,171 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478d27bdd3414e26832a1a88fb06ce00, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731991298919 2024-11-19T04:41:43,171 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 118242453ade48a5bc0504593ff17023, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731991300103 2024-11-19T04:41:43,171 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90b0dc91ea774bcba3a75a72963cb057, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731991300103 2024-11-19T04:41:43,172 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29fdfaf021b74e34a0df233ee86aceec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731991302282 2024-11-19T04:41:43,172 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d7654f5d3642b1806bdd6149659ac4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731991302282 2024-11-19T04:41:43,179 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:43,180 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:43,180 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/126c5e30b0304da0b9e9914b7aed1e04 is 50, key is test_row_0/B:col10/1731991302282/Put/seqid=0 2024-11-19T04:41:43,180 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/79a0add74caa4d94bb7de298fc7dd574 is 50, key is test_row_0/A:col10/1731991302282/Put/seqid=0 2024-11-19T04:41:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742135_1311 (size=12629) 2024-11-19T04:41:43,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742134_1310 (size=12629) 2024-11-19T04:41:43,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-19T04:41:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T04:41:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/06ac0fae8eba4822acac12e01cba1118 is 50, key is test_row_0/A:col10/1731991302332/Put/seqid=0 2024-11-19T04:41:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742136_1312 (size=12151) 2024-11-19T04:41:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991363497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991363497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991363498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991363499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991363501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,595 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/79a0add74caa4d94bb7de298fc7dd574 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/79a0add74caa4d94bb7de298fc7dd574 2024-11-19T04:41:43,595 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/126c5e30b0304da0b9e9914b7aed1e04 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/126c5e30b0304da0b9e9914b7aed1e04 2024-11-19T04:41:43,600 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 79a0add74caa4d94bb7de298fc7dd574(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:43,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:43,600 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991303169; duration=0sec 2024-11-19T04:41:43,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:43,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:43,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:43,602 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 126c5e30b0304da0b9e9914b7aed1e04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:43,602 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:43,602 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991303169; duration=0sec 2024-11-19T04:41:43,602 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:43,602 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:43,603 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:43,603 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:43,603 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:43,603 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/26b4fba33da4480bb8a2e547b1d1d0a5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ceffa79a7cca40ef82967a0126141507, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1b7de0a7721d4e00a03a2c78d57a1547] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.0 K 2024-11-19T04:41:43,603 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26b4fba33da4480bb8a2e547b1d1d0a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731991298919 2024-11-19T04:41:43,604 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceffa79a7cca40ef82967a0126141507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731991300103 2024-11-19T04:41:43,605 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b7de0a7721d4e00a03a2c78d57a1547, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731991302282 2024-11-19T04:41:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991363626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991363626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991363626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,634 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:43,635 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ca0cd7d27d4e4692aeaa2f606644b9d0 is 50, key is test_row_0/C:col10/1731991302282/Put/seqid=0 2024-11-19T04:41:43,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742137_1313 (size=12629) 2024-11-19T04:41:43,644 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ca0cd7d27d4e4692aeaa2f606644b9d0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca0cd7d27d4e4692aeaa2f606644b9d0 2024-11-19T04:41:43,649 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into ca0cd7d27d4e4692aeaa2f606644b9d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:43,649 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:43,649 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991303169; duration=0sec 2024-11-19T04:41:43,650 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:43,650 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:43,731 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/06ac0fae8eba4822acac12e01cba1118 2024-11-19T04:41:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e948c968029a439c8875205c8ae53e8f is 50, key is test_row_0/B:col10/1731991302332/Put/seqid=0 2024-11-19T04:41:43,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742138_1314 (size=12151) 2024-11-19T04:41:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991363836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991363837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991363837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,144 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e948c968029a439c8875205c8ae53e8f 2024-11-19T04:41:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991364147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991364148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991364148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/dbaf685abd9e429bb6bead1916df8f12 is 50, key is test_row_0/C:col10/1731991302332/Put/seqid=0 2024-11-19T04:41:44,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742139_1315 (size=12151) 2024-11-19T04:41:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:44,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991364517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991364517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,521 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 641e0316dd7be3ea1908e5148ffce278 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:44,521 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 641e0316dd7be3ea1908e5148ffce278 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,557 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/dbaf685abd9e429bb6bead1916df8f12 2024-11-19T04:41:44,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/06ac0fae8eba4822acac12e01cba1118 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/06ac0fae8eba4822acac12e01cba1118 2024-11-19T04:41:44,575 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/06ac0fae8eba4822acac12e01cba1118, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T04:41:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e948c968029a439c8875205c8ae53e8f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e948c968029a439c8875205c8ae53e8f 2024-11-19T04:41:44,581 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e948c968029a439c8875205c8ae53e8f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T04:41:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/dbaf685abd9e429bb6bead1916df8f12 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dbaf685abd9e429bb6bead1916df8f12 2024-11-19T04:41:44,587 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dbaf685abd9e429bb6bead1916df8f12, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T04:41:44,588 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 641e0316dd7be3ea1908e5148ffce278 in 1271ms, sequenceid=253, compaction requested=false 2024-11-19T04:41:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:44,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-19T04:41:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-19T04:41:44,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T04:41:44,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-11-19T04:41:44,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3550 sec 2024-11-19T04:41:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:44,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:41:44,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:44,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:44,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:44,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/7e178521f6d749aa81a33b1feee7ca49 is 50, key is test_row_0/A:col10/1731991304658/Put/seqid=0 2024-11-19T04:41:44,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742140_1316 (size=12301) 2024-11-19T04:41:44,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/7e178521f6d749aa81a33b1feee7ca49 2024-11-19T04:41:44,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/f0de0fbce41149c98f40f3bcdde33fb2 is 50, key is test_row_0/B:col10/1731991304658/Put/seqid=0 2024-11-19T04:41:44,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742141_1317 (size=12301) 2024-11-19T04:41:44,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/f0de0fbce41149c98f40f3bcdde33fb2 2024-11-19T04:41:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991364691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a427d903985c4f8f883a71e23914289a is 50, key is test_row_0/C:col10/1731991304658/Put/seqid=0 2024-11-19T04:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991364696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742142_1318 (size=12301) 2024-11-19T04:41:44,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a427d903985c4f8f883a71e23914289a 2024-11-19T04:41:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991364698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/7e178521f6d749aa81a33b1feee7ca49 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/7e178521f6d749aa81a33b1feee7ca49 2024-11-19T04:41:44,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/7e178521f6d749aa81a33b1feee7ca49, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T04:41:44,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/f0de0fbce41149c98f40f3bcdde33fb2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f0de0fbce41149c98f40f3bcdde33fb2 2024-11-19T04:41:44,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f0de0fbce41149c98f40f3bcdde33fb2, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T04:41:44,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a427d903985c4f8f883a71e23914289a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a427d903985c4f8f883a71e23914289a 2024-11-19T04:41:44,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a427d903985c4f8f883a71e23914289a, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T04:41:44,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 641e0316dd7be3ea1908e5148ffce278 in 60ms, sequenceid=268, compaction requested=true 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:44,719 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:44,719 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:44,721 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:44,721 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:44,721 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:44,721 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:44,721 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:44,721 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:44,721 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/79a0add74caa4d94bb7de298fc7dd574, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/06ac0fae8eba4822acac12e01cba1118, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/7e178521f6d749aa81a33b1feee7ca49] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.2 K 2024-11-19T04:41:44,721 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/126c5e30b0304da0b9e9914b7aed1e04, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e948c968029a439c8875205c8ae53e8f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f0de0fbce41149c98f40f3bcdde33fb2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.2 K 2024-11-19T04:41:44,722 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a0add74caa4d94bb7de298fc7dd574, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731991302282 2024-11-19T04:41:44,722 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 126c5e30b0304da0b9e9914b7aed1e04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731991302282 2024-11-19T04:41:44,722 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ac0fae8eba4822acac12e01cba1118, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731991302332 2024-11-19T04:41:44,722 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e948c968029a439c8875205c8ae53e8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731991302332 2024-11-19T04:41:44,722 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f0de0fbce41149c98f40f3bcdde33fb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731991303491 2024-11-19T04:41:44,722 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e178521f6d749aa81a33b1feee7ca49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731991303491 2024-11-19T04:41:44,733 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:44,734 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/1638e9f72ff44643bd085798687edf1c is 50, key is test_row_0/B:col10/1731991304658/Put/seqid=0 2024-11-19T04:41:44,736 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:44,737 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/424bb0091d6f4f58a16dffd60758c80e is 50, key is test_row_0/A:col10/1731991304658/Put/seqid=0 2024-11-19T04:41:44,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742143_1319 (size=12881) 2024-11-19T04:41:44,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742144_1320 (size=12881) 2024-11-19T04:41:44,754 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/424bb0091d6f4f58a16dffd60758c80e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/424bb0091d6f4f58a16dffd60758c80e 2024-11-19T04:41:44,759 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 424bb0091d6f4f58a16dffd60758c80e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:44,759 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:44,759 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991304719; duration=0sec 2024-11-19T04:41:44,759 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:44,759 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:44,759 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:44,760 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:44,760 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:44,760 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:44,760 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca0cd7d27d4e4692aeaa2f606644b9d0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dbaf685abd9e429bb6bead1916df8f12, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a427d903985c4f8f883a71e23914289a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.2 K 2024-11-19T04:41:44,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0cd7d27d4e4692aeaa2f606644b9d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731991302282 2024-11-19T04:41:44,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbaf685abd9e429bb6bead1916df8f12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731991302332 2024-11-19T04:41:44,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a427d903985c4f8f883a71e23914289a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731991303491 2024-11-19T04:41:44,771 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:44,771 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/2222ab68110645b09cf793e492764461 is 50, key is test_row_0/C:col10/1731991304658/Put/seqid=0 2024-11-19T04:41:44,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742145_1321 (size=12881) 2024-11-19T04:41:44,783 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/2222ab68110645b09cf793e492764461 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/2222ab68110645b09cf793e492764461 2024-11-19T04:41:44,788 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into 2222ab68110645b09cf793e492764461(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:44,789 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:44,789 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991304719; duration=0sec 2024-11-19T04:41:44,789 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:44,789 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:44,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T04:41:44,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:44,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991304666, max timestamp=1731991304809Num uniques 859; 2024-11-19T04:41:44,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991304666, max timestamp=1731991304809Num uniques 859; 2024-11-19T04:41:44,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:44,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:44,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991304666, max timestamp=1731991304809Num uniques 859; 2024-11-19T04:41:44,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:41:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:44,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/a74c2e7c23cc4283b51ff6ff089fa656 is 50, key is test_row_0/A:col10/1731991304810/Put/seqid=0 2024-11-19T04:41:44,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742146_1322 (size=14741) 2024-11-19T04:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731991304837, max timestamp=1731991304860Num uniques 859; 2024-11-19T04:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731991304837, max timestamp=1731991304860Num uniques 859; 2024-11-19T04:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:44,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:44,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991304837, max timestamp=1731991304860Num uniques 859; 2024-11-19T04:41:44,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:44,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731991304889, max timestamp=1731991304908Num uniques 1068; 2024-11-19T04:41:44,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T04:41:44,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731991304889, max timestamp=1731991304908Num uniques 1068; 2024-11-19T04:41:44,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T04:41:44,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731991304889, max timestamp=1731991304908Num uniques 1068; 2024-11-19T04:41:44,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-19T04:41:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991364947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:44,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731991304937, max timestamp=1731991304950Num uniques 1077; 2024-11-19T04:41:44,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731991304937, max timestamp=1731991304950Num uniques 1077; 2024-11-19T04:41:44,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T04:41:44,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T04:41:44,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731991304937, max timestamp=1731991304950Num uniques 1077; 2024-11-19T04:41:44,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T04:41:45,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731991304989, max timestamp=1731991305005Num uniques 1086; 2024-11-19T04:41:45,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T04:41:45,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731991304989, max timestamp=1731991305005Num uniques 1086; 2024-11-19T04:41:45,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T04:41:45,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731991304989, max timestamp=1731991305005Num uniques 1086; 2024-11-19T04:41:45,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-19T04:41:45,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731991305047, max timestamp=1731991305063Num uniques 1095; 2024-11-19T04:41:45,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T04:41:45,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731991305047, max timestamp=1731991305063Num uniques 1095; 2024-11-19T04:41:45,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T04:41:45,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731991305047, max timestamp=1731991305063Num uniques 1095; 2024-11-19T04:41:45,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-19T04:41:45,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731991305091, max timestamp=1731991305106Num uniques 1054; 2024-11-19T04:41:45,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-19T04:41:45,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731991305091, max timestamp=1731991305106Num uniques 1054; 2024-11-19T04:41:45,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-19T04:41:45,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731991305091, max timestamp=1731991305106Num uniques 1054; 2024-11-19T04:41:45,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-19T04:41:45,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731991305133, max timestamp=1731991305144Num uniques 1013; 2024-11-19T04:41:45,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-19T04:41:45,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731991305133, max timestamp=1731991305144Num uniques 1013; 2024-11-19T04:41:45,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-19T04:41:45,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731991305133, max timestamp=1731991305144Num uniques 1013; 2024-11-19T04:41:45,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-19T04:41:45,152 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/1638e9f72ff44643bd085798687edf1c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/1638e9f72ff44643bd085798687edf1c 2024-11-19T04:41:45,157 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 1638e9f72ff44643bd085798687edf1c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:45,157 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:45,157 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991304719; duration=0sec 2024-11-19T04:41:45,157 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:45,157 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:45,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731991305165, max timestamp=1731991305182Num uniques 1072; 2024-11-19T04:41:45,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731991305165, max timestamp=1731991305182Num uniques 1072; 2024-11-19T04:41:45,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T04:41:45,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T04:41:45,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731991305165, max timestamp=1731991305182Num uniques 1072; 2024-11-19T04:41:45,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-19T04:41:45,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731991305209, max timestamp=1731991305220Num uniques 1081; 2024-11-19T04:41:45,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=12866, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-11-19T04:41:45,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731991305209, max timestamp=1731991305220Num uniques 1081; 2024-11-19T04:41:45,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=12866, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-11-19T04:41:45,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731991305209, max timestamp=1731991305220Num uniques 1081; 2024-11-19T04:41:45,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=12866, suffix heap size=140392,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=281 2024-11-19T04:41:45,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/a74c2e7c23cc4283b51ff6ff089fa656 2024-11-19T04:41:45,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e039dcc3240e4b93bd5f6d15f6f4f337 is 50, key is test_row_0/B:col10/1731991304810/Put/seqid=0 2024-11-19T04:41:45,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991365263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991365266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731991305254, max timestamp=1731991305267Num uniques 1140; 2024-11-19T04:41:45,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731991305254, max timestamp=1731991305267Num uniques 1140; 2024-11-19T04:41:45,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-19T04:41:45,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-19T04:41:45,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731991305254, max timestamp=1731991305267Num uniques 1140; 2024-11-19T04:41:45,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=8700, suffix heap size=146528,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=190 2024-11-19T04:41:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742147_1323 (size=12301) 2024-11-19T04:41:45,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731991305336, max timestamp=1731991305361Num uniques 1049; 2024-11-19T04:41:45,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T04:41:45,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731991305336, max timestamp=1731991305361Num uniques 1049; 2024-11-19T04:41:45,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T04:41:45,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731991305336, max timestamp=1731991305361Num uniques 1049; 2024-11-19T04:41:45,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-19T04:41:45,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731991305392, max timestamp=1731991305410Num uniques 1058; 2024-11-19T04:41:45,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731991305392, max timestamp=1731991305410Num uniques 1058; 2024-11-19T04:41:45,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731991305392, max timestamp=1731991305410Num uniques 1058; 2024-11-19T04:41:45,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-11-19T04:41:45,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-11-19T04:41:45,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-11-19T04:41:45,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991365448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991365450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731991305441, max timestamp=1731991305453Num uniques 1117; 2024-11-19T04:41:45,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-11-19T04:41:45,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731991305441, max timestamp=1731991305453Num uniques 1117; 2024-11-19T04:41:45,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-11-19T04:41:45,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731991305441, max timestamp=1731991305453Num uniques 1117; 2024-11-19T04:41:45,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-11-19T04:41:45,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731991305519, max timestamp=1731991305549Num uniques 1026; 2024-11-19T04:41:45,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-19T04:41:45,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731991305519, max timestamp=1731991305549Num uniques 1026; 2024-11-19T04:41:45,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731991305519, max timestamp=1731991305549Num uniques 1026; 2024-11-19T04:41:45,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-19T04:41:45,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-19T04:41:45,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731991305585, max timestamp=1731991305597Num uniques 1035; 2024-11-19T04:41:45,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-11-19T04:41:45,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731991305585, max timestamp=1731991305597Num uniques 1035; 2024-11-19T04:41:45,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-11-19T04:41:45,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731991305585, max timestamp=1731991305597Num uniques 1035; 2024-11-19T04:41:45,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=8470, suffix heap size=135608,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=185 2024-11-19T04:41:45,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731991305631, max timestamp=1731991305663Num uniques 1044; 2024-11-19T04:41:45,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731991305631, max timestamp=1731991305663Num uniques 1044; 2024-11-19T04:41:45,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731991305631, max timestamp=1731991305663Num uniques 1044; 2024-11-19T04:41:45,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-11-19T04:41:45,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-11-19T04:41:45,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-11-19T04:41:45,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e039dcc3240e4b93bd5f6d15f6f4f337 2024-11-19T04:41:45,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/9ff94c0b24a74b0dbb5420e76a00aea3 is 50, key is test_row_0/C:col10/1731991304810/Put/seqid=0 2024-11-19T04:41:45,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742148_1324 (size=12301) 2024-11-19T04:41:45,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/9ff94c0b24a74b0dbb5420e76a00aea3 2024-11-19T04:41:45,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/a74c2e7c23cc4283b51ff6ff089fa656 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/a74c2e7c23cc4283b51ff6ff089fa656 2024-11-19T04:41:45,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/a74c2e7c23cc4283b51ff6ff089fa656, entries=200, sequenceid=292, filesize=14.4 K 2024-11-19T04:41:45,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/e039dcc3240e4b93bd5f6d15f6f4f337 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e039dcc3240e4b93bd5f6d15f6f4f337 2024-11-19T04:41:45,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e039dcc3240e4b93bd5f6d15f6f4f337, entries=150, sequenceid=292, filesize=12.0 K 2024-11-19T04:41:45,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/9ff94c0b24a74b0dbb5420e76a00aea3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/9ff94c0b24a74b0dbb5420e76a00aea3 2024-11-19T04:41:45,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731991305692, max timestamp=1731991305702Num uniques 1053; 2024-11-19T04:41:45,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-19T04:41:45,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731991305692, max timestamp=1731991305702Num uniques 1053; 2024-11-19T04:41:45,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-19T04:41:45,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731991305692, max timestamp=1731991305702Num uniques 1053; 2024-11-19T04:41:45,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-19T04:41:45,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/9ff94c0b24a74b0dbb5420e76a00aea3, entries=150, sequenceid=292, filesize=12.0 K 2024-11-19T04:41:45,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=53.67 KB/54960 for 641e0316dd7be3ea1908e5148ffce278 in 902ms, sequenceid=292, compaction requested=false 2024-11-19T04:41:45,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:45,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=60.38 KB heapSize=161.72 KB 2024-11-19T04:41:45,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:45,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:45,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:45,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:45,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:45,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:45,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/8166009e4c3c4866a7a74704c8aad70f is 50, key is test_row_0/A:col10/1731991305712/Put/seqid=0 2024-11-19T04:41:45,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742149_1325 (size=19621) 2024-11-19T04:41:45,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991305743, max timestamp=1731991305757Num uniques 859; 2024-11-19T04:41:45,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:45,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991305743, max timestamp=1731991305757Num uniques 859; 2024-11-19T04:41:45,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991305743, max timestamp=1731991305757Num uniques 859; 2024-11-19T04:41:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:45,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:45,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991365790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991365792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991365797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991365907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991365907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991365917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991366117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991366117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/8166009e4c3c4866a7a74704c8aad70f 2024-11-19T04:41:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991366127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/a733fefbfa824254aac43eda55fcb579 is 50, key is test_row_0/B:col10/1731991305712/Put/seqid=0 2024-11-19T04:41:46,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742150_1326 (size=12301) 2024-11-19T04:41:46,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/a733fefbfa824254aac43eda55fcb579 2024-11-19T04:41:46,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/bf2abd1902974bd5903b8255137391ad is 50, key is test_row_0/C:col10/1731991305712/Put/seqid=0 2024-11-19T04:41:46,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742151_1327 (size=12301) 2024-11-19T04:41:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-19T04:41:46,407 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:46,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-19T04:41:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T04:41:46,410 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:46,410 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:46,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991366428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991366428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991366436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T04:41:46,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991366526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,531 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 641e0316dd7be3ea1908e5148ffce278 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991366537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,542 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 641e0316dd7be3ea1908e5148ffce278 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/bf2abd1902974bd5903b8255137391ad 2024-11-19T04:41:46,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T04:41:46,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:46,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:46,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:46,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:46,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/8166009e4c3c4866a7a74704c8aad70f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/8166009e4c3c4866a7a74704c8aad70f 2024-11-19T04:41:46,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:46,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/8166009e4c3c4866a7a74704c8aad70f, entries=300, sequenceid=593, filesize=19.2 K 2024-11-19T04:41:46,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/a733fefbfa824254aac43eda55fcb579 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/a733fefbfa824254aac43eda55fcb579 2024-11-19T04:41:46,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/a733fefbfa824254aac43eda55fcb579, entries=150, sequenceid=593, filesize=12.0 K 2024-11-19T04:41:46,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/bf2abd1902974bd5903b8255137391ad as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bf2abd1902974bd5903b8255137391ad 2024-11-19T04:41:46,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bf2abd1902974bd5903b8255137391ad, entries=150, sequenceid=593, filesize=12.0 K 2024-11-19T04:41:46,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~161.67 KB/165552, currentSize=127.47 KB/130530 for 641e0316dd7be3ea1908e5148ffce278 in 866ms, sequenceid=593, compaction requested=true 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:46,579 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:46,579 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:46,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:46,580 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:46,580 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:46,580 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:46,580 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:46,580 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:46,580 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:46,580 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/424bb0091d6f4f58a16dffd60758c80e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/a74c2e7c23cc4283b51ff6ff089fa656, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/8166009e4c3c4866a7a74704c8aad70f] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=46.1 K 2024-11-19T04:41:46,580 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/1638e9f72ff44643bd085798687edf1c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e039dcc3240e4b93bd5f6d15f6f4f337, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/a733fefbfa824254aac43eda55fcb579] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.6 K 2024-11-19T04:41:46,581 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424bb0091d6f4f58a16dffd60758c80e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731991303491 2024-11-19T04:41:46,581 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1638e9f72ff44643bd085798687edf1c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731991303491 2024-11-19T04:41:46,581 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e039dcc3240e4b93bd5f6d15f6f4f337, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731991304666 2024-11-19T04:41:46,581 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a74c2e7c23cc4283b51ff6ff089fa656, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731991304666 2024-11-19T04:41:46,581 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8166009e4c3c4866a7a74704c8aad70f, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1731991305701 2024-11-19T04:41:46,581 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a733fefbfa824254aac43eda55fcb579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1731991305709 2024-11-19T04:41:46,602 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:46,603 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/abb69d2f23d849a79679bd6d65a2b233 is 50, key is test_row_0/B:col10/1731991305712/Put/seqid=0 2024-11-19T04:41:46,604 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:46,605 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/f8eac1614b6b4d7b911311e43b45e1b0 is 50, key is test_row_0/A:col10/1731991305712/Put/seqid=0 2024-11-19T04:41:46,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742152_1328 (size=12983) 2024-11-19T04:41:46,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742153_1329 (size=12983) 2024-11-19T04:41:46,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-19T04:41:46,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:46,719 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T04:41:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/2c1ca5aa8a0e43e2bce613a34d86dc00 is 50, key is test_row_0/A:col10/1731991305779/Put/seqid=0 2024-11-19T04:41:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T04:41:46,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742154_1330 (size=12301) 2024-11-19T04:41:46,728 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/2c1ca5aa8a0e43e2bce613a34d86dc00 2024-11-19T04:41:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/220316600e74460ebe68f0b6da04f520 is 50, key is test_row_0/B:col10/1731991305779/Put/seqid=0 2024-11-19T04:41:46,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742155_1331 (size=12301) 2024-11-19T04:41:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:46,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991366956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991366956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991366959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:46,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,013 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/abb69d2f23d849a79679bd6d65a2b233 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/abb69d2f23d849a79679bd6d65a2b233 2024-11-19T04:41:47,019 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into abb69d2f23d849a79679bd6d65a2b233(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:47,019 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:47,019 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991306579; duration=0sec 2024-11-19T04:41:47,019 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:47,019 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:47,019 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:47,019 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/f8eac1614b6b4d7b911311e43b45e1b0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f8eac1614b6b4d7b911311e43b45e1b0 2024-11-19T04:41:47,021 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:47,021 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:47,021 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:47,021 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/2222ab68110645b09cf793e492764461, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/9ff94c0b24a74b0dbb5420e76a00aea3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bf2abd1902974bd5903b8255137391ad] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.6 K 2024-11-19T04:41:47,022 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 2222ab68110645b09cf793e492764461, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731991303491 2024-11-19T04:41:47,022 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ff94c0b24a74b0dbb5420e76a00aea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731991304666 2024-11-19T04:41:47,023 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2abd1902974bd5903b8255137391ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1731991305709 2024-11-19T04:41:47,024 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into f8eac1614b6b4d7b911311e43b45e1b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:47,024 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:47,024 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991306579; duration=0sec 2024-11-19T04:41:47,024 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:47,025 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:47,032 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:47,032 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/de06f54c9ae7466bbe2e518d61da4011 is 50, key is test_row_0/C:col10/1731991305712/Put/seqid=0 2024-11-19T04:41:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T04:41:47,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742156_1332 (size=12983) 2024-11-19T04:41:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991367067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991367067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991367067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,142 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/220316600e74460ebe68f0b6da04f520 2024-11-19T04:41:47,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/61bdb482af4943b1b9d310ef168999f6 is 50, key is test_row_0/C:col10/1731991305779/Put/seqid=0 2024-11-19T04:41:47,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742157_1333 (size=12301) 2024-11-19T04:41:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991367277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991367277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991367278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,451 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/de06f54c9ae7466bbe2e518d61da4011 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/de06f54c9ae7466bbe2e518d61da4011 2024-11-19T04:41:47,455 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into de06f54c9ae7466bbe2e518d61da4011(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:47,455 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:47,455 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991306579; duration=0sec 2024-11-19T04:41:47,456 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:47,456 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T04:41:47,554 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/61bdb482af4943b1b9d310ef168999f6 2024-11-19T04:41:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/2c1ca5aa8a0e43e2bce613a34d86dc00 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2c1ca5aa8a0e43e2bce613a34d86dc00 2024-11-19T04:41:47,562 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2c1ca5aa8a0e43e2bce613a34d86dc00, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T04:41:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/220316600e74460ebe68f0b6da04f520 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/220316600e74460ebe68f0b6da04f520 2024-11-19T04:41:47,566 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/220316600e74460ebe68f0b6da04f520, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T04:41:47,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/61bdb482af4943b1b9d310ef168999f6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/61bdb482af4943b1b9d310ef168999f6 2024-11-19T04:41:47,571 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/61bdb482af4943b1b9d310ef168999f6, entries=150, sequenceid=628, filesize=12.0 K 2024-11-19T04:41:47,572 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 641e0316dd7be3ea1908e5148ffce278 in 854ms, sequenceid=628, compaction requested=false 2024-11-19T04:41:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-19T04:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-19T04:41:47,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-19T04:41:47,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1640 sec 2024-11-19T04:41:47,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.1700 sec 2024-11-19T04:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:47,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:41:47,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:47,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:47,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/af897ea3d70f428aa5f742bad54c3464 is 50, key is test_row_0/A:col10/1731991306958/Put/seqid=0 2024-11-19T04:41:47,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742158_1334 (size=12301) 2024-11-19T04:41:47,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/af897ea3d70f428aa5f742bad54c3464 2024-11-19T04:41:47,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/b38c601484594939a0f95d56ceaf8781 is 50, key is test_row_0/B:col10/1731991306958/Put/seqid=0 2024-11-19T04:41:47,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742159_1335 (size=12301) 2024-11-19T04:41:47,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731991307608, max timestamp=1731991307649Num uniques 859; 2024-11-19T04:41:47,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:47,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731991307608, max timestamp=1731991307649Num uniques 859; 2024-11-19T04:41:47,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:41:47,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731991307608, max timestamp=1731991307649Num uniques 859; 2024-11-19T04:41:47,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T04:41:47,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991367681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991367681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991367682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991367796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991367796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991367797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991368007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991368007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991368008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/b38c601484594939a0f95d56ceaf8781 2024-11-19T04:41:48,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ca908d9a3fd649399b7b17897606cfe0 is 50, key is test_row_0/C:col10/1731991306958/Put/seqid=0 2024-11-19T04:41:48,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742160_1336 (size=12301) 2024-11-19T04:41:48,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991368317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991368317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991368317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ca908d9a3fd649399b7b17897606cfe0 2024-11-19T04:41:48,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/af897ea3d70f428aa5f742bad54c3464 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/af897ea3d70f428aa5f742bad54c3464 2024-11-19T04:41:48,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/af897ea3d70f428aa5f742bad54c3464, entries=150, sequenceid=643, filesize=12.0 K 2024-11-19T04:41:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/b38c601484594939a0f95d56ceaf8781 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b38c601484594939a0f95d56ceaf8781 2024-11-19T04:41:48,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b38c601484594939a0f95d56ceaf8781, entries=150, sequenceid=643, filesize=12.0 K 2024-11-19T04:41:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/ca908d9a3fd649399b7b17897606cfe0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca908d9a3fd649399b7b17897606cfe0 2024-11-19T04:41:48,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca908d9a3fd649399b7b17897606cfe0, entries=150, sequenceid=643, filesize=12.0 K 2024-11-19T04:41:48,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=122.60 KB/125546 for 641e0316dd7be3ea1908e5148ffce278 in 853ms, sequenceid=643, compaction requested=true 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:48,442 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:48,442 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:48,443 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:48,443 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:48,443 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:48,443 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:48,443 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:48,443 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:48,443 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/abb69d2f23d849a79679bd6d65a2b233, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/220316600e74460ebe68f0b6da04f520, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b38c601484594939a0f95d56ceaf8781] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.7 K 2024-11-19T04:41:48,443 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f8eac1614b6b4d7b911311e43b45e1b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2c1ca5aa8a0e43e2bce613a34d86dc00, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/af897ea3d70f428aa5f742bad54c3464] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.7 K 2024-11-19T04:41:48,444 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting abb69d2f23d849a79679bd6d65a2b233, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1731991305709 2024-11-19T04:41:48,444 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8eac1614b6b4d7b911311e43b45e1b0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1731991305709 2024-11-19T04:41:48,444 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 220316600e74460ebe68f0b6da04f520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731991305779 2024-11-19T04:41:48,444 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c1ca5aa8a0e43e2bce613a34d86dc00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731991305779 2024-11-19T04:41:48,444 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting af897ea3d70f428aa5f742bad54c3464, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731991306954 2024-11-19T04:41:48,444 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b38c601484594939a0f95d56ceaf8781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731991306954 2024-11-19T04:41:48,451 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:48,451 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/66a7d838e953405d94d92e8fb757c8fa is 50, key is test_row_0/B:col10/1731991306958/Put/seqid=0 2024-11-19T04:41:48,454 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:48,455 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/1602f9a4803e4fb993e01aaa8bc820b0 is 50, key is test_row_0/A:col10/1731991306958/Put/seqid=0 2024-11-19T04:41:48,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742161_1337 (size=13085) 2024-11-19T04:41:48,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742162_1338 (size=13085) 2024-11-19T04:41:48,479 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/1602f9a4803e4fb993e01aaa8bc820b0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/1602f9a4803e4fb993e01aaa8bc820b0 2024-11-19T04:41:48,483 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 1602f9a4803e4fb993e01aaa8bc820b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:48,483 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:48,483 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991308442; duration=0sec 2024-11-19T04:41:48,483 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:48,483 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:48,483 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:48,484 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:48,484 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:48,484 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:48,484 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/de06f54c9ae7466bbe2e518d61da4011, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/61bdb482af4943b1b9d310ef168999f6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca908d9a3fd649399b7b17897606cfe0] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.7 K 2024-11-19T04:41:48,485 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting de06f54c9ae7466bbe2e518d61da4011, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1731991305709 2024-11-19T04:41:48,485 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61bdb482af4943b1b9d310ef168999f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=628, earliestPutTs=1731991305779 2024-11-19T04:41:48,485 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca908d9a3fd649399b7b17897606cfe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731991306954 2024-11-19T04:41:48,492 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:48,493 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/14547592efeb409180e9a3dea4d6317b is 50, key is test_row_0/C:col10/1731991306958/Put/seqid=0 2024-11-19T04:41:48,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742163_1339 (size=13085) 2024-11-19T04:41:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T04:41:48,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:48,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-19T04:41:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T04:41:48,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:48,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:48,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T04:41:48,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-19T04:41:48,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:48,713 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=122.60 KB heapSize=344.62 KB 2024-11-19T04:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:48,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/b91ccae083294f9194a4ed678119bbe0 is 50, key is test_row_0/A:col10/1731991307680/Put/seqid=0 2024-11-19T04:41:48,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742164_1340 (size=12301) 2024-11-19T04:41:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:48,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991368844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991368845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991368847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,866 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/66a7d838e953405d94d92e8fb757c8fa as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/66a7d838e953405d94d92e8fb757c8fa 2024-11-19T04:41:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T04:41:48,878 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 66a7d838e953405d94d92e8fb757c8fa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:48,878 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:48,878 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991308442; duration=0sec 2024-11-19T04:41:48,878 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:48,878 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:48,910 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/14547592efeb409180e9a3dea4d6317b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/14547592efeb409180e9a3dea4d6317b 2024-11-19T04:41:48,914 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into 14547592efeb409180e9a3dea4d6317b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:48,914 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:48,915 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991308442; duration=0sec 2024-11-19T04:41:48,915 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:48,915 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991368956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991368957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991368957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:48,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,126 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/b91ccae083294f9194a4ed678119bbe0 2024-11-19T04:41:49,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/98686f32f0484025bad0dcdb63f383eb is 50, key is test_row_0/B:col10/1731991307680/Put/seqid=0 2024-11-19T04:41:49,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742165_1341 (size=12301) 2024-11-19T04:41:49,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991369166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:49,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991369166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991369167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T04:41:49,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991369477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:49,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991369477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991369477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:49,542 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/98686f32f0484025bad0dcdb63f383eb 2024-11-19T04:41:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/1db72321afab4fca96c29d12b6cbd220 is 50, key is test_row_0/C:col10/1731991307680/Put/seqid=0 2024-11-19T04:41:49,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742166_1342 (size=12301) 2024-11-19T04:41:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T04:41:49,955 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/1db72321afab4fca96c29d12b6cbd220 2024-11-19T04:41:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/b91ccae083294f9194a4ed678119bbe0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/b91ccae083294f9194a4ed678119bbe0 2024-11-19T04:41:49,963 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/b91ccae083294f9194a4ed678119bbe0, entries=150, sequenceid=678, filesize=12.0 K 2024-11-19T04:41:49,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/98686f32f0484025bad0dcdb63f383eb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/98686f32f0484025bad0dcdb63f383eb 2024-11-19T04:41:49,968 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/98686f32f0484025bad0dcdb63f383eb, entries=150, sequenceid=678, filesize=12.0 K 2024-11-19T04:41:49,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/1db72321afab4fca96c29d12b6cbd220 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1db72321afab4fca96c29d12b6cbd220 2024-11-19T04:41:49,982 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1db72321afab4fca96c29d12b6cbd220, entries=150, sequenceid=678, filesize=12.0 K 2024-11-19T04:41:49,983 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~122.60 KB/125546, heapSize ~344.57 KB/352840, currentSize=60.38 KB/61830 for 641e0316dd7be3ea1908e5148ffce278 in 1270ms, sequenceid=678, compaction requested=false 2024-11-19T04:41:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-19T04:41:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-19T04:41:49,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T04:41:49,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-19T04:41:49,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.4300 sec 2024-11-19T04:41:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:49,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:41:49,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:49,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:49,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:49,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:49,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:49,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:49,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3bddcd46deb54480891d9829f2176606 is 50, key is test_row_0/A:col10/1731991309989/Put/seqid=0 2024-11-19T04:41:50,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742167_1343 (size=12301) 2024-11-19T04:41:50,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991370016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991370017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991370018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991370127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991370127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991370127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991370337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991370337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991370337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3bddcd46deb54480891d9829f2176606 2024-11-19T04:41:50,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/b02fb49612bc40079e417fb2bdc3312f is 50, key is test_row_0/B:col10/1731991309989/Put/seqid=0 2024-11-19T04:41:50,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742168_1344 (size=12301) 2024-11-19T04:41:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51246 deadline: 1731991370547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,550 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 641e0316dd7be3ea1908e5148ffce278 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51230 deadline: 1731991370567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,571 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 641e0316dd7be3ea1908e5148ffce278 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:41:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991370647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991370647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991370647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-19T04:41:50,707 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:50,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-19T04:41:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T04:41:50,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:50,710 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:50,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T04:41:50,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/b02fb49612bc40079e417fb2bdc3312f 2024-11-19T04:41:50,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/95543d1b51aa4e6081fa8c42444a4bae is 50, key is test_row_0/C:col10/1731991309989/Put/seqid=0 2024-11-19T04:41:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742169_1345 (size=12301) 2024-11-19T04:41:50,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T04:41:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:50,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T04:41:51,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:51,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T04:41:51,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991371157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991371157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:51,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T04:41:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991371166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:51,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:51,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/95543d1b51aa4e6081fa8c42444a4bae 2024-11-19T04:41:51,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3bddcd46deb54480891d9829f2176606 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3bddcd46deb54480891d9829f2176606 2024-11-19T04:41:51,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3bddcd46deb54480891d9829f2176606, entries=150, sequenceid=694, filesize=12.0 K 2024-11-19T04:41:51,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/b02fb49612bc40079e417fb2bdc3312f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b02fb49612bc40079e417fb2bdc3312f 2024-11-19T04:41:51,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b02fb49612bc40079e417fb2bdc3312f, entries=150, sequenceid=694, filesize=12.0 K 2024-11-19T04:41:51,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/95543d1b51aa4e6081fa8c42444a4bae as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/95543d1b51aa4e6081fa8c42444a4bae 2024-11-19T04:41:51,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/95543d1b51aa4e6081fa8c42444a4bae, entries=150, sequenceid=694, filesize=12.0 K 2024-11-19T04:41:51,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 641e0316dd7be3ea1908e5148ffce278 in 1273ms, sequenceid=694, compaction requested=true 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:51,263 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:51,263 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:51,264 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:51,264 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:51,264 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:51,264 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:51,264 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,264 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,264 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/66a7d838e953405d94d92e8fb757c8fa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/98686f32f0484025bad0dcdb63f383eb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b02fb49612bc40079e417fb2bdc3312f] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.8 K 2024-11-19T04:41:51,264 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/1602f9a4803e4fb993e01aaa8bc820b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/b91ccae083294f9194a4ed678119bbe0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3bddcd46deb54480891d9829f2176606] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.8 K 2024-11-19T04:41:51,265 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a7d838e953405d94d92e8fb757c8fa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731991306954 2024-11-19T04:41:51,265 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1602f9a4803e4fb993e01aaa8bc820b0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731991306954 2024-11-19T04:41:51,265 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 98686f32f0484025bad0dcdb63f383eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731991307671 2024-11-19T04:41:51,265 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b91ccae083294f9194a4ed678119bbe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731991307671 2024-11-19T04:41:51,266 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b02fb49612bc40079e417fb2bdc3312f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731991308843 2024-11-19T04:41:51,266 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bddcd46deb54480891d9829f2176606, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731991308843 2024-11-19T04:41:51,274 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:51,275 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/56b9238e05c84e2b91d7a6f30dc93b65 is 50, key is test_row_0/A:col10/1731991309989/Put/seqid=0 2024-11-19T04:41:51,275 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:51,276 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/6f44ecf4eb8f41f89d83a7eb19551f95 is 50, key is test_row_0/B:col10/1731991309989/Put/seqid=0 2024-11-19T04:41:51,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742170_1346 (size=13187) 2024-11-19T04:41:51,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742171_1347 (size=13187) 2024-11-19T04:41:51,288 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/56b9238e05c84e2b91d7a6f30dc93b65 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/56b9238e05c84e2b91d7a6f30dc93b65 2024-11-19T04:41:51,288 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/6f44ecf4eb8f41f89d83a7eb19551f95 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/6f44ecf4eb8f41f89d83a7eb19551f95 2024-11-19T04:41:51,293 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 56b9238e05c84e2b91d7a6f30dc93b65(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:51,293 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991311263; duration=0sec 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:51,293 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 6f44ecf4eb8f41f89d83a7eb19551f95(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:51,293 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991311263; duration=0sec 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:51,293 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:51,294 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:51,294 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:51,294 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,294 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/14547592efeb409180e9a3dea4d6317b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1db72321afab4fca96c29d12b6cbd220, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/95543d1b51aa4e6081fa8c42444a4bae] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.8 K 2024-11-19T04:41:51,294 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14547592efeb409180e9a3dea4d6317b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731991306954 2024-11-19T04:41:51,295 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db72321afab4fca96c29d12b6cbd220, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1731991307671 2024-11-19T04:41:51,295 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95543d1b51aa4e6081fa8c42444a4bae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731991308843 2024-11-19T04:41:51,301 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:51,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/dfa1e0685c164050828ec9ea392c0406 is 50, key is test_row_0/C:col10/1731991309989/Put/seqid=0 2024-11-19T04:41:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742172_1348 (size=13187) 2024-11-19T04:41:51,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-19T04:41:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,322 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T04:41:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:51,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/6f993739c3bd4412a4762881377825e0 is 50, key is test_row_0/A:col10/1731991310017/Put/seqid=0 2024-11-19T04:41:51,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742173_1349 (size=12301) 2024-11-19T04:41:51,332 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/6f993739c3bd4412a4762881377825e0 2024-11-19T04:41:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T04:41:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/0e1238d3bb34418dbc87ae94067538e9 is 50, key is test_row_0/B:col10/1731991310017/Put/seqid=0 2024-11-19T04:41:51,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742174_1350 (size=12301) 2024-11-19T04:41:51,344 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/0e1238d3bb34418dbc87ae94067538e9 2024-11-19T04:41:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/c1517d079b354c9080b459e4462ef067 is 50, key is test_row_0/C:col10/1731991310017/Put/seqid=0 2024-11-19T04:41:51,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742175_1351 (size=12301) 2024-11-19T04:41:51,709 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/dfa1e0685c164050828ec9ea392c0406 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dfa1e0685c164050828ec9ea392c0406 2024-11-19T04:41:51,714 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into dfa1e0685c164050828ec9ea392c0406(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:51,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:51,714 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991311263; duration=0sec 2024-11-19T04:41:51,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:51,714 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:51,756 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/c1517d079b354c9080b459e4462ef067 2024-11-19T04:41:51,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/6f993739c3bd4412a4762881377825e0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6f993739c3bd4412a4762881377825e0 2024-11-19T04:41:51,764 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6f993739c3bd4412a4762881377825e0, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T04:41:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/0e1238d3bb34418dbc87ae94067538e9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0e1238d3bb34418dbc87ae94067538e9 2024-11-19T04:41:51,768 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0e1238d3bb34418dbc87ae94067538e9, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T04:41:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/c1517d079b354c9080b459e4462ef067 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/c1517d079b354c9080b459e4462ef067 2024-11-19T04:41:51,772 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/c1517d079b354c9080b459e4462ef067, entries=150, sequenceid=714, filesize=12.0 K 2024-11-19T04:41:51,773 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 641e0316dd7be3ea1908e5148ffce278 in 451ms, sequenceid=714, compaction requested=false 2024-11-19T04:41:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-19T04:41:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-19T04:41:51,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T04:41:51,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0650 sec 2024-11-19T04:41:51,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.0690 sec 2024-11-19T04:41:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T04:41:51,847 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:51,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-19T04:41:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T04:41:51,850 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:51,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:51,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T04:41:52,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-19T04:41:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-19T04:41:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-19T04:41:52,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T04:41:52,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-19T04:41:52,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 160 msec 2024-11-19T04:41:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-19T04:41:52,168 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:52,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:41:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-19T04:41:52,170 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:41:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T04:41:52,172 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:41:52,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:41:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:52,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:41:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:52,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/2533ae4214db43b5abfd6a3bd5a3f298 is 50, key is test_row_0/A:col10/1731991312170/Put/seqid=0 2024-11-19T04:41:52,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742176_1352 (size=14741) 2024-11-19T04:41:52,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991312196, max timestamp=1731991312211Num uniques 859; 2024-11-19T04:41:52,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991312196, max timestamp=1731991312211Num uniques 859; 2024-11-19T04:41:52,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:52,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:52,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991312196, max timestamp=1731991312211Num uniques 859; 2024-11-19T04:41:52,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:41:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991372243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991372251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991372252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T04:41:52,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T04:41:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991372366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991372367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991372367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T04:41:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T04:41:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991372577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991372577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991372577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/2533ae4214db43b5abfd6a3bd5a3f298 2024-11-19T04:41:52,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/476d66e236b44e05928b2fc803d95281 is 50, key is test_row_0/B:col10/1731991312170/Put/seqid=0 2024-11-19T04:41:52,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742177_1353 (size=12301) 2024-11-19T04:41:52,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T04:41:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T04:41:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T04:41:52,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991372887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991372888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991372888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:52,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T04:41:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:52,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:41:53,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/476d66e236b44e05928b2fc803d95281 2024-11-19T04:41:53,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/fbe9b57d035b4aa3a2f4ce44ef291830 is 50, key is test_row_0/C:col10/1731991312170/Put/seqid=0 2024-11-19T04:41:53,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742178_1354 (size=12301) 2024-11-19T04:41:53,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=726 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/fbe9b57d035b4aa3a2f4ce44ef291830 2024-11-19T04:41:53,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/2533ae4214db43b5abfd6a3bd5a3f298 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2533ae4214db43b5abfd6a3bd5a3f298 2024-11-19T04:41:53,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2533ae4214db43b5abfd6a3bd5a3f298, entries=200, sequenceid=726, filesize=14.4 K 2024-11-19T04:41:53,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/476d66e236b44e05928b2fc803d95281 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/476d66e236b44e05928b2fc803d95281 2024-11-19T04:41:53,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/476d66e236b44e05928b2fc803d95281, entries=150, sequenceid=726, filesize=12.0 K 2024-11-19T04:41:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/fbe9b57d035b4aa3a2f4ce44ef291830 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/fbe9b57d035b4aa3a2f4ce44ef291830 2024-11-19T04:41:53,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/fbe9b57d035b4aa3a2f4ce44ef291830, entries=150, sequenceid=726, filesize=12.0 K 2024-11-19T04:41:53,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 641e0316dd7be3ea1908e5148ffce278 in 888ms, sequenceid=726, compaction requested=true 2024-11-19T04:41:53,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:53,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:53,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:53,065 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:53,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:53,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:53,065 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:53,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:53,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:53,066 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:53,066 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:53,066 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:53,066 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/56b9238e05c84e2b91d7a6f30dc93b65, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6f993739c3bd4412a4762881377825e0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2533ae4214db43b5abfd6a3bd5a3f298] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=39.3 K 2024-11-19T04:41:53,068 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:53,068 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:53,068 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:53,068 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/6f44ecf4eb8f41f89d83a7eb19551f95, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0e1238d3bb34418dbc87ae94067538e9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/476d66e236b44e05928b2fc803d95281] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.9 K 2024-11-19T04:41:53,069 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b9238e05c84e2b91d7a6f30dc93b65, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731991308843 2024-11-19T04:41:53,070 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f44ecf4eb8f41f89d83a7eb19551f95, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731991308843 2024-11-19T04:41:53,070 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f993739c3bd4412a4762881377825e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731991310012 2024-11-19T04:41:53,074 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1238d3bb34418dbc87ae94067538e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731991310012 2024-11-19T04:41:53,074 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2533ae4214db43b5abfd6a3bd5a3f298, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1731991312170 2024-11-19T04:41:53,075 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 476d66e236b44e05928b2fc803d95281, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1731991312170 2024-11-19T04:41:53,089 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:53,090 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/d03eec95027141148e62fefe446338e2 is 50, key is test_row_0/A:col10/1731991312170/Put/seqid=0 2024-11-19T04:41:53,091 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:53,091 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/3703c8c2c15a44e6bd8dd225f5afc490 is 50, key is test_row_0/B:col10/1731991312170/Put/seqid=0 2024-11-19T04:41:53,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-19T04:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:53,093 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T04:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:53,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:41:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/9b582ac543eb403e8a206001c173f1a2 is 50, key is test_row_0/A:col10/1731991312241/Put/seqid=0 2024-11-19T04:41:53,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742180_1356 (size=13289) 2024-11-19T04:41:53,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742179_1355 (size=13289) 2024-11-19T04:41:53,153 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/d03eec95027141148e62fefe446338e2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/d03eec95027141148e62fefe446338e2 2024-11-19T04:41:53,165 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into d03eec95027141148e62fefe446338e2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:53,165 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:53,165 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991313064; duration=0sec 2024-11-19T04:41:53,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742181_1357 (size=12301) 2024-11-19T04:41:53,165 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:53,166 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:53,166 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:53,169 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:53,169 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:53,169 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=762 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/9b582ac543eb403e8a206001c173f1a2 2024-11-19T04:41:53,169 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:53,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dfa1e0685c164050828ec9ea392c0406, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/c1517d079b354c9080b459e4462ef067, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/fbe9b57d035b4aa3a2f4ce44ef291830] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=36.9 K 2024-11-19T04:41:53,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa1e0685c164050828ec9ea392c0406, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=694, earliestPutTs=1731991308843 2024-11-19T04:41:53,171 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1517d079b354c9080b459e4462ef067, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731991310012 2024-11-19T04:41:53,172 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe9b57d035b4aa3a2f4ce44ef291830, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1731991312170 2024-11-19T04:41:53,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/76251a7eaf744d9a8547205a9f5a240b is 50, key is test_row_0/B:col10/1731991312241/Put/seqid=0 2024-11-19T04:41:53,199 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:53,199 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/bdbc93b5259a44e3a056b459acdec6d5 is 50, key is test_row_0/C:col10/1731991312170/Put/seqid=0 2024-11-19T04:41:53,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742182_1358 (size=12301) 2024-11-19T04:41:53,223 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=762 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/76251a7eaf744d9a8547205a9f5a240b 2024-11-19T04:41:53,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/516b6938ff124f91bc0c8802baa7a75f is 50, key is test_row_0/C:col10/1731991312241/Put/seqid=0 2024-11-19T04:41:53,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742183_1359 (size=13289) 2024-11-19T04:41:53,255 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/bdbc93b5259a44e3a056b459acdec6d5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bdbc93b5259a44e3a056b459acdec6d5 2024-11-19T04:41:53,263 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into bdbc93b5259a44e3a056b459acdec6d5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:53,263 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:53,263 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991313065; duration=0sec 2024-11-19T04:41:53,263 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:53,263 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:41:53,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742184_1360 (size=12301) 2024-11-19T04:41:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T04:41:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:53,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. as already flushing 2024-11-19T04:41:53,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991373413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:53,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:53,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991373418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991373419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:53,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:53,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51206 deadline: 1731991373527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:53,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51260 deadline: 1731991373527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:41:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51282 deadline: 1731991373528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:41:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:41:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:41:53,542 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/3703c8c2c15a44e6bd8dd225f5afc490 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/3703c8c2c15a44e6bd8dd225f5afc490 2024-11-19T04:41:53,547 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 3703c8c2c15a44e6bd8dd225f5afc490(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:53,547 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:53,547 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991313064; duration=0sec 2024-11-19T04:41:53,547 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:53,547 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:53,655 INFO [Thread-1242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1242. 2024-11-19T04:41:53,655 INFO [Thread-1236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1236. 2024-11-19T04:41:53,655 DEBUG [Thread-1242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,655 DEBUG [Thread-1236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,656 DEBUG [Thread-1242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,656 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,656 DEBUG [Thread-1242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,659 INFO [Thread-1244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1244. 2024-11-19T04:41:53,659 DEBUG [Thread-1244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,659 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,659 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,660 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,661 INFO [Thread-1240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1240. 2024-11-19T04:41:53,661 DEBUG [Thread-1240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,661 DEBUG [Thread-1240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,661 DEBUG [Thread-1240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,661 INFO [Thread-1238 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1238. 2024-11-19T04:41:53,662 DEBUG [Thread-1238 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,662 DEBUG [Thread-1238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,662 DEBUG [Thread-1238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,667 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=762 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/516b6938ff124f91bc0c8802baa7a75f 2024-11-19T04:41:53,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/9b582ac543eb403e8a206001c173f1a2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9b582ac543eb403e8a206001c173f1a2 2024-11-19T04:41:53,684 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9b582ac543eb403e8a206001c173f1a2, entries=150, sequenceid=762, filesize=12.0 K 2024-11-19T04:41:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/76251a7eaf744d9a8547205a9f5a240b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/76251a7eaf744d9a8547205a9f5a240b 2024-11-19T04:41:53,689 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/76251a7eaf744d9a8547205a9f5a240b, entries=150, sequenceid=762, filesize=12.0 K 2024-11-19T04:41:53,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/516b6938ff124f91bc0c8802baa7a75f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/516b6938ff124f91bc0c8802baa7a75f 2024-11-19T04:41:53,693 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/516b6938ff124f91bc0c8802baa7a75f, entries=150, sequenceid=762, filesize=12.0 K 2024-11-19T04:41:53,694 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 641e0316dd7be3ea1908e5148ffce278 in 601ms, sequenceid=762, compaction requested=false 2024-11-19T04:41:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-19T04:41:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-19T04:41:53,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-19T04:41:53,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5230 sec 2024-11-19T04:41:53,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.5290 sec 2024-11-19T04:41:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:41:53,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:41:53,738 INFO [Thread-1233 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1233. 2024-11-19T04:41:53,738 DEBUG [Thread-1233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,738 INFO [Thread-1229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1229. 2024-11-19T04:41:53,738 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,738 DEBUG [Thread-1229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,738 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,739 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,739 INFO [Thread-1231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1231. 2024-11-19T04:41:53,739 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:41:53,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,739 DEBUG [Thread-1231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:41:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:53,739 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:41:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:53,739 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:41:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:41:53,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:41:53,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:41:53,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3523a147518d422096c9a1319a9deb3e is 50, key is test_row_0/A:col10/1731991313737/Put/seqid=0 2024-11-19T04:41:53,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742185_1361 (size=12301) 2024-11-19T04:41:54,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=778 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3523a147518d422096c9a1319a9deb3e 2024-11-19T04:41:54,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5631d7047af14d8eaa510723a200c54b is 50, key is test_row_0/B:col10/1731991313737/Put/seqid=0 2024-11-19T04:41:54,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742186_1362 (size=12301) 2024-11-19T04:41:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-19T04:41:54,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:41:54,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=778 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5631d7047af14d8eaa510723a200c54b 2024-11-19T04:41:54,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/5476ba30c8db4f3eb64e6d7bccd6e784 is 50, key is test_row_0/C:col10/1731991313737/Put/seqid=0 2024-11-19T04:41:54,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742187_1363 (size=12301) 2024-11-19T04:41:54,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=778 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/5476ba30c8db4f3eb64e6d7bccd6e784 2024-11-19T04:41:54,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/3523a147518d422096c9a1319a9deb3e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3523a147518d422096c9a1319a9deb3e 2024-11-19T04:41:54,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3523a147518d422096c9a1319a9deb3e, entries=150, sequenceid=778, filesize=12.0 K 2024-11-19T04:41:54,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/5631d7047af14d8eaa510723a200c54b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5631d7047af14d8eaa510723a200c54b 2024-11-19T04:41:54,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5631d7047af14d8eaa510723a200c54b, entries=150, sequenceid=778, filesize=12.0 K 2024-11-19T04:41:54,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/5476ba30c8db4f3eb64e6d7bccd6e784 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/5476ba30c8db4f3eb64e6d7bccd6e784 2024-11-19T04:41:54,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/5476ba30c8db4f3eb64e6d7bccd6e784, entries=150, sequenceid=778, filesize=12.0 K 2024-11-19T04:41:54,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 641e0316dd7be3ea1908e5148ffce278 in 1241ms, sequenceid=778, compaction requested=true 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 641e0316dd7be3ea1908e5148ffce278:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:41:54,979 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:54,979 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/A is initiating minor compaction (all files) 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/B is initiating minor compaction (all files) 2024-11-19T04:41:54,980 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/A in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:54,980 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/B in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:54,980 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/d03eec95027141148e62fefe446338e2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9b582ac543eb403e8a206001c173f1a2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3523a147518d422096c9a1319a9deb3e] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=37.0 K 2024-11-19T04:41:54,980 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/3703c8c2c15a44e6bd8dd225f5afc490, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/76251a7eaf744d9a8547205a9f5a240b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5631d7047af14d8eaa510723a200c54b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=37.0 K 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03eec95027141148e62fefe446338e2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1731991312170 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 3703c8c2c15a44e6bd8dd225f5afc490, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1731991312170 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b582ac543eb403e8a206001c173f1a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=762, earliestPutTs=1731991312241 2024-11-19T04:41:54,980 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 76251a7eaf744d9a8547205a9f5a240b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=762, earliestPutTs=1731991312241 2024-11-19T04:41:54,981 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5631d7047af14d8eaa510723a200c54b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1731991313737 2024-11-19T04:41:54,981 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3523a147518d422096c9a1319a9deb3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1731991313737 2024-11-19T04:41:54,987 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#A#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:54,987 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:54,987 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/10084accd117421c939ef15adc22bcce is 50, key is test_row_0/A:col10/1731991313737/Put/seqid=0 2024-11-19T04:41:54,988 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/26b1c7b0edb44b71a68c81a95b820bf3 is 50, key is test_row_0/B:col10/1731991313737/Put/seqid=0 2024-11-19T04:41:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742188_1364 (size=13391) 2024-11-19T04:41:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742189_1365 (size=13391) 2024-11-19T04:41:55,398 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/10084accd117421c939ef15adc22bcce as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/10084accd117421c939ef15adc22bcce 2024-11-19T04:41:55,398 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/26b1c7b0edb44b71a68c81a95b820bf3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/26b1c7b0edb44b71a68c81a95b820bf3 2024-11-19T04:41:55,402 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/B of 641e0316dd7be3ea1908e5148ffce278 into 26b1c7b0edb44b71a68c81a95b820bf3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:55,402 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/A of 641e0316dd7be3ea1908e5148ffce278 into 10084accd117421c939ef15adc22bcce(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:55,402 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/A, priority=13, startTime=1731991314979; duration=0sec 2024-11-19T04:41:55,402 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/B, priority=13, startTime=1731991314979; duration=0sec 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:A 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:41:55,402 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:B 2024-11-19T04:41:55,403 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:41:55,403 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 641e0316dd7be3ea1908e5148ffce278/C is initiating minor compaction (all files) 2024-11-19T04:41:55,403 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 641e0316dd7be3ea1908e5148ffce278/C in TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:41:55,403 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bdbc93b5259a44e3a056b459acdec6d5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/516b6938ff124f91bc0c8802baa7a75f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/5476ba30c8db4f3eb64e6d7bccd6e784] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp, totalSize=37.0 K 2024-11-19T04:41:55,403 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdbc93b5259a44e3a056b459acdec6d5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=726, earliestPutTs=1731991312170 2024-11-19T04:41:55,403 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516b6938ff124f91bc0c8802baa7a75f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=762, earliestPutTs=1731991312241 2024-11-19T04:41:55,404 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5476ba30c8db4f3eb64e6d7bccd6e784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=778, earliestPutTs=1731991313737 2024-11-19T04:41:55,409 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 641e0316dd7be3ea1908e5148ffce278#C#compaction#314 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:41:55,410 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a3d82e6edae34eef9c80750392f9ab31 is 50, key is test_row_0/C:col10/1731991313737/Put/seqid=0 2024-11-19T04:41:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742190_1366 (size=13391) 2024-11-19T04:41:55,818 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/a3d82e6edae34eef9c80750392f9ab31 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a3d82e6edae34eef9c80750392f9ab31 2024-11-19T04:41:55,822 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 641e0316dd7be3ea1908e5148ffce278/C of 641e0316dd7be3ea1908e5148ffce278 into a3d82e6edae34eef9c80750392f9ab31(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:41:55,822 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 641e0316dd7be3ea1908e5148ffce278: 2024-11-19T04:41:55,822 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., storeName=641e0316dd7be3ea1908e5148ffce278/C, priority=13, startTime=1731991314979; duration=0sec 2024-11-19T04:41:55,822 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:41:55,822 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 641e0316dd7be3ea1908e5148ffce278:C 2024-11-19T04:42:00,608 INFO [Thread-1227 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1227. 2024-11-19T04:42:00,609 DEBUG [Thread-1227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:00,609 DEBUG [Thread-1227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:00,609 DEBUG [Thread-1227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:00,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:00,629 INFO [Thread-1225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1225. 2024-11-19T04:42:00,629 DEBUG [Thread-1225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:00,629 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:00,629 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:00,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 219 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 201 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2706 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2673 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2694 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2686 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2664 2024-11-19T04:42:00,629 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T04:42:00,629 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:42:00,629 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:42:00,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:00,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:00,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:00,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T04:42:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T04:42:00,633 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991320632"}]},"ts":"1731991320632"} 2024-11-19T04:42:00,634 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T04:42:00,634 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T04:42:00,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:42:00,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, UNASSIGN}] 2024-11-19T04:42:00,636 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, UNASSIGN 2024-11-19T04:42:00,637 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=641e0316dd7be3ea1908e5148ffce278, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:00,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, UNASSIGN because future has completed 2024-11-19T04:42:00,638 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:42:00,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:42:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T04:42:00,790 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:42:00,790 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:42:00,790 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 641e0316dd7be3ea1908e5148ffce278, disabling compactions & flushes 2024-11-19T04:42:00,790 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:42:00,790 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:42:00,790 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. after waiting 0 ms 2024-11-19T04:42:00,790 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:42:00,790 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 641e0316dd7be3ea1908e5148ffce278 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-19T04:42:00,791 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=A 2024-11-19T04:42:00,791 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:00,791 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=B 2024-11-19T04:42:00,791 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:00,791 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 641e0316dd7be3ea1908e5148ffce278, store=C 2024-11-19T04:42:00,791 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:00,794 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/028d38b336d64a76a854d5c8763b4162 is 50, key is test_row_1/A:col10/1731991320607/Put/seqid=0 2024-11-19T04:42:00,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742191_1367 (size=9857) 2024-11-19T04:42:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T04:42:01,198 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/028d38b336d64a76a854d5c8763b4162 2024-11-19T04:42:01,204 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/7840769b3160497aaa83cd39c38eeb8b is 50, key is test_row_1/B:col10/1731991320607/Put/seqid=0 2024-11-19T04:42:01,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742192_1368 (size=9857) 2024-11-19T04:42:01,207 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/7840769b3160497aaa83cd39c38eeb8b 2024-11-19T04:42:01,212 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/7e1111355d18473b847cf72534b59bdb is 50, key is test_row_1/C:col10/1731991320607/Put/seqid=0 2024-11-19T04:42:01,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742193_1369 (size=9857) 2024-11-19T04:42:01,219 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T04:42:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T04:42:01,616 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/7e1111355d18473b847cf72534b59bdb 2024-11-19T04:42:01,619 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/A/028d38b336d64a76a854d5c8763b4162 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/028d38b336d64a76a854d5c8763b4162 2024-11-19T04:42:01,622 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/028d38b336d64a76a854d5c8763b4162, entries=100, sequenceid=786, filesize=9.6 K 2024-11-19T04:42:01,623 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/B/7840769b3160497aaa83cd39c38eeb8b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/7840769b3160497aaa83cd39c38eeb8b 2024-11-19T04:42:01,626 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/7840769b3160497aaa83cd39c38eeb8b, entries=100, sequenceid=786, filesize=9.6 K 2024-11-19T04:42:01,627 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/.tmp/C/7e1111355d18473b847cf72534b59bdb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/7e1111355d18473b847cf72534b59bdb 2024-11-19T04:42:01,630 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/7e1111355d18473b847cf72534b59bdb, entries=100, sequenceid=786, filesize=9.6 K 2024-11-19T04:42:01,631 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 641e0316dd7be3ea1908e5148ffce278 in 841ms, sequenceid=786, compaction requested=false 2024-11-19T04:42:01,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/48ee7fc311e64483a41de90b3bd587d1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4521691ef35b4df08b56e259c1dbdd51, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3a07e1a1b65a468c8c315f9330a5c0f8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4cf2a87810a64c28895981d68f5678b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6bbf11afda84450a9e12a75f8e671d29, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3f4b305c665c41cda5b2606a9f9f032a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ebf24646edba4cc990df3d370a4a46ce, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f29c082c88a447329a64db85429eb719, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ed5a8ed8bc8b45c488db073cd49557aa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9e9c055fb5ee424dbb7e024020c6b162, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/59626a72153348b5a21ea730de0ca076, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/478d27bdd3414e26832a1a88fb06ce00, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/90b0dc91ea774bcba3a75a72963cb057, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/79a0add74caa4d94bb7de298fc7dd574, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/29fdfaf021b74e34a0df233ee86aceec, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/06ac0fae8eba4822acac12e01cba1118, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/424bb0091d6f4f58a16dffd60758c80e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/7e178521f6d749aa81a33b1feee7ca49, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/a74c2e7c23cc4283b51ff6ff089fa656, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/8166009e4c3c4866a7a74704c8aad70f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f8eac1614b6b4d7b911311e43b45e1b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2c1ca5aa8a0e43e2bce613a34d86dc00, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/1602f9a4803e4fb993e01aaa8bc820b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/af897ea3d70f428aa5f742bad54c3464, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/b91ccae083294f9194a4ed678119bbe0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/56b9238e05c84e2b91d7a6f30dc93b65, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3bddcd46deb54480891d9829f2176606, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6f993739c3bd4412a4762881377825e0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2533ae4214db43b5abfd6a3bd5a3f298, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/d03eec95027141148e62fefe446338e2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9b582ac543eb403e8a206001c173f1a2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3523a147518d422096c9a1319a9deb3e] to archive 2024-11-19T04:42:01,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:42:01,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/48ee7fc311e64483a41de90b3bd587d1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/48ee7fc311e64483a41de90b3bd587d1 2024-11-19T04:42:01,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4521691ef35b4df08b56e259c1dbdd51 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4521691ef35b4df08b56e259c1dbdd51 2024-11-19T04:42:01,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3a07e1a1b65a468c8c315f9330a5c0f8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3a07e1a1b65a468c8c315f9330a5c0f8 2024-11-19T04:42:01,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4cf2a87810a64c28895981d68f5678b0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/4cf2a87810a64c28895981d68f5678b0 2024-11-19T04:42:01,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6bbf11afda84450a9e12a75f8e671d29 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6bbf11afda84450a9e12a75f8e671d29 2024-11-19T04:42:01,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3f4b305c665c41cda5b2606a9f9f032a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3f4b305c665c41cda5b2606a9f9f032a 2024-11-19T04:42:01,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ebf24646edba4cc990df3d370a4a46ce to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ebf24646edba4cc990df3d370a4a46ce 2024-11-19T04:42:01,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f29c082c88a447329a64db85429eb719 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f29c082c88a447329a64db85429eb719 2024-11-19T04:42:01,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ed5a8ed8bc8b45c488db073cd49557aa to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/ed5a8ed8bc8b45c488db073cd49557aa 2024-11-19T04:42:01,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9e9c055fb5ee424dbb7e024020c6b162 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9e9c055fb5ee424dbb7e024020c6b162 2024-11-19T04:42:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/59626a72153348b5a21ea730de0ca076 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/59626a72153348b5a21ea730de0ca076 2024-11-19T04:42:01,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/478d27bdd3414e26832a1a88fb06ce00 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/478d27bdd3414e26832a1a88fb06ce00 2024-11-19T04:42:01,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/90b0dc91ea774bcba3a75a72963cb057 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/90b0dc91ea774bcba3a75a72963cb057 2024-11-19T04:42:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/79a0add74caa4d94bb7de298fc7dd574 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/79a0add74caa4d94bb7de298fc7dd574 2024-11-19T04:42:01,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/29fdfaf021b74e34a0df233ee86aceec to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/29fdfaf021b74e34a0df233ee86aceec 2024-11-19T04:42:01,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/06ac0fae8eba4822acac12e01cba1118 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/06ac0fae8eba4822acac12e01cba1118 2024-11-19T04:42:01,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/424bb0091d6f4f58a16dffd60758c80e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/424bb0091d6f4f58a16dffd60758c80e 2024-11-19T04:42:01,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/7e178521f6d749aa81a33b1feee7ca49 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/7e178521f6d749aa81a33b1feee7ca49 2024-11-19T04:42:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/a74c2e7c23cc4283b51ff6ff089fa656 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/a74c2e7c23cc4283b51ff6ff089fa656 2024-11-19T04:42:01,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/8166009e4c3c4866a7a74704c8aad70f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/8166009e4c3c4866a7a74704c8aad70f 2024-11-19T04:42:01,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f8eac1614b6b4d7b911311e43b45e1b0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/f8eac1614b6b4d7b911311e43b45e1b0 2024-11-19T04:42:01,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2c1ca5aa8a0e43e2bce613a34d86dc00 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2c1ca5aa8a0e43e2bce613a34d86dc00 2024-11-19T04:42:01,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/1602f9a4803e4fb993e01aaa8bc820b0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/1602f9a4803e4fb993e01aaa8bc820b0 2024-11-19T04:42:01,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/af897ea3d70f428aa5f742bad54c3464 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/af897ea3d70f428aa5f742bad54c3464 2024-11-19T04:42:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/b91ccae083294f9194a4ed678119bbe0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/b91ccae083294f9194a4ed678119bbe0 2024-11-19T04:42:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/56b9238e05c84e2b91d7a6f30dc93b65 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/56b9238e05c84e2b91d7a6f30dc93b65 2024-11-19T04:42:01,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3bddcd46deb54480891d9829f2176606 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3bddcd46deb54480891d9829f2176606 2024-11-19T04:42:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6f993739c3bd4412a4762881377825e0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/6f993739c3bd4412a4762881377825e0 2024-11-19T04:42:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2533ae4214db43b5abfd6a3bd5a3f298 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/2533ae4214db43b5abfd6a3bd5a3f298 2024-11-19T04:42:01,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/d03eec95027141148e62fefe446338e2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/d03eec95027141148e62fefe446338e2 2024-11-19T04:42:01,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9b582ac543eb403e8a206001c173f1a2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/9b582ac543eb403e8a206001c173f1a2 2024-11-19T04:42:01,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3523a147518d422096c9a1319a9deb3e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/3523a147518d422096c9a1319a9deb3e 2024-11-19T04:42:01,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5c39bb40b25a476f8f103a0c6a9e2525, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5e2b5b5a16a748d39b49622e9bcbe848, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/099aa420889f46b2ac0b9d80c4fc7d71, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f8a67afeb93c4dfcb6603519dfcf0263, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/d98ac311fb6143e490729e8d33e1dc14, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e3f7182c4a7c4a1d9410b09823cc5e7f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c356e9a471734213a6868e81fa462e38, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e791a8b5f96d4ecbab1fa435e1632f58, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e0e36442357c4993a006885e0daf71b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/ae25f318e552400e84d71cdc38c94ecc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0600d81834964b749e403a8ef2fc673c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/934f8c97126e4d7cbadb605ed3e8e402, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/118242453ade48a5bc0504593ff17023, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/126c5e30b0304da0b9e9914b7aed1e04, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c3d7654f5d3642b1806bdd6149659ac4, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e948c968029a439c8875205c8ae53e8f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/1638e9f72ff44643bd085798687edf1c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f0de0fbce41149c98f40f3bcdde33fb2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e039dcc3240e4b93bd5f6d15f6f4f337, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/abb69d2f23d849a79679bd6d65a2b233, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/a733fefbfa824254aac43eda55fcb579, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/220316600e74460ebe68f0b6da04f520, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/66a7d838e953405d94d92e8fb757c8fa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b38c601484594939a0f95d56ceaf8781, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/98686f32f0484025bad0dcdb63f383eb, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/6f44ecf4eb8f41f89d83a7eb19551f95, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b02fb49612bc40079e417fb2bdc3312f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0e1238d3bb34418dbc87ae94067538e9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/3703c8c2c15a44e6bd8dd225f5afc490, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/476d66e236b44e05928b2fc803d95281, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/76251a7eaf744d9a8547205a9f5a240b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5631d7047af14d8eaa510723a200c54b] to archive 2024-11-19T04:42:01,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:42:01,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5c39bb40b25a476f8f103a0c6a9e2525 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5c39bb40b25a476f8f103a0c6a9e2525 2024-11-19T04:42:01,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5e2b5b5a16a748d39b49622e9bcbe848 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5e2b5b5a16a748d39b49622e9bcbe848 2024-11-19T04:42:01,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/099aa420889f46b2ac0b9d80c4fc7d71 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/099aa420889f46b2ac0b9d80c4fc7d71 2024-11-19T04:42:01,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f8a67afeb93c4dfcb6603519dfcf0263 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f8a67afeb93c4dfcb6603519dfcf0263 2024-11-19T04:42:01,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/d98ac311fb6143e490729e8d33e1dc14 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/d98ac311fb6143e490729e8d33e1dc14 2024-11-19T04:42:01,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e3f7182c4a7c4a1d9410b09823cc5e7f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e3f7182c4a7c4a1d9410b09823cc5e7f 2024-11-19T04:42:01,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c356e9a471734213a6868e81fa462e38 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c356e9a471734213a6868e81fa462e38 2024-11-19T04:42:01,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e791a8b5f96d4ecbab1fa435e1632f58 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e791a8b5f96d4ecbab1fa435e1632f58 2024-11-19T04:42:01,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e0e36442357c4993a006885e0daf71b0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e0e36442357c4993a006885e0daf71b0 2024-11-19T04:42:01,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/ae25f318e552400e84d71cdc38c94ecc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/ae25f318e552400e84d71cdc38c94ecc 2024-11-19T04:42:01,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0600d81834964b749e403a8ef2fc673c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0600d81834964b749e403a8ef2fc673c 2024-11-19T04:42:01,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/934f8c97126e4d7cbadb605ed3e8e402 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/934f8c97126e4d7cbadb605ed3e8e402 2024-11-19T04:42:01,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/118242453ade48a5bc0504593ff17023 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/118242453ade48a5bc0504593ff17023 2024-11-19T04:42:01,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/126c5e30b0304da0b9e9914b7aed1e04 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/126c5e30b0304da0b9e9914b7aed1e04 2024-11-19T04:42:01,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c3d7654f5d3642b1806bdd6149659ac4 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/c3d7654f5d3642b1806bdd6149659ac4 2024-11-19T04:42:01,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e948c968029a439c8875205c8ae53e8f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e948c968029a439c8875205c8ae53e8f 2024-11-19T04:42:01,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/1638e9f72ff44643bd085798687edf1c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/1638e9f72ff44643bd085798687edf1c 2024-11-19T04:42:01,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f0de0fbce41149c98f40f3bcdde33fb2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/f0de0fbce41149c98f40f3bcdde33fb2 2024-11-19T04:42:01,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e039dcc3240e4b93bd5f6d15f6f4f337 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/e039dcc3240e4b93bd5f6d15f6f4f337 2024-11-19T04:42:01,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/abb69d2f23d849a79679bd6d65a2b233 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/abb69d2f23d849a79679bd6d65a2b233 2024-11-19T04:42:01,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/a733fefbfa824254aac43eda55fcb579 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/a733fefbfa824254aac43eda55fcb579 2024-11-19T04:42:01,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/220316600e74460ebe68f0b6da04f520 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/220316600e74460ebe68f0b6da04f520 2024-11-19T04:42:01,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/66a7d838e953405d94d92e8fb757c8fa to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/66a7d838e953405d94d92e8fb757c8fa 2024-11-19T04:42:01,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b38c601484594939a0f95d56ceaf8781 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b38c601484594939a0f95d56ceaf8781 2024-11-19T04:42:01,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/98686f32f0484025bad0dcdb63f383eb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/98686f32f0484025bad0dcdb63f383eb 2024-11-19T04:42:01,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/6f44ecf4eb8f41f89d83a7eb19551f95 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/6f44ecf4eb8f41f89d83a7eb19551f95 2024-11-19T04:42:01,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b02fb49612bc40079e417fb2bdc3312f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/b02fb49612bc40079e417fb2bdc3312f 2024-11-19T04:42:01,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0e1238d3bb34418dbc87ae94067538e9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/0e1238d3bb34418dbc87ae94067538e9 2024-11-19T04:42:01,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/3703c8c2c15a44e6bd8dd225f5afc490 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/3703c8c2c15a44e6bd8dd225f5afc490 2024-11-19T04:42:01,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/476d66e236b44e05928b2fc803d95281 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/476d66e236b44e05928b2fc803d95281 2024-11-19T04:42:01,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/76251a7eaf744d9a8547205a9f5a240b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/76251a7eaf744d9a8547205a9f5a240b 2024-11-19T04:42:01,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5631d7047af14d8eaa510723a200c54b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/5631d7047af14d8eaa510723a200c54b 2024-11-19T04:42:01,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/330f74150b84412783f5a40240134275, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/b997f9c92a1048e0b64a5906dcb2b01c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/98c1831bba9a4a89a92376508e32012e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/718476832b614784872fb97b31580c18, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/20252e11b1a148feaf42d365f8846e19, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/6894afe7829b4f80b248e8f1c481d4a7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/37483eb4a4bb4aca9a09f3a4680e2882, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a5f686b98a6646f5b13ca2989eeee919, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/490a0259774348f883892e3c00e0b470, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/96a5534fe29a48ed805640843f24bd93, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/26b4fba33da4480bb8a2e547b1d1d0a5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/e5f1141754ec4089a1761f805ef9f412, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ceffa79a7cca40ef82967a0126141507, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca0cd7d27d4e4692aeaa2f606644b9d0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1b7de0a7721d4e00a03a2c78d57a1547, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dbaf685abd9e429bb6bead1916df8f12, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/2222ab68110645b09cf793e492764461, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a427d903985c4f8f883a71e23914289a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/9ff94c0b24a74b0dbb5420e76a00aea3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/de06f54c9ae7466bbe2e518d61da4011, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bf2abd1902974bd5903b8255137391ad, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/61bdb482af4943b1b9d310ef168999f6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/14547592efeb409180e9a3dea4d6317b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca908d9a3fd649399b7b17897606cfe0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1db72321afab4fca96c29d12b6cbd220, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dfa1e0685c164050828ec9ea392c0406, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/95543d1b51aa4e6081fa8c42444a4bae, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/c1517d079b354c9080b459e4462ef067, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bdbc93b5259a44e3a056b459acdec6d5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/fbe9b57d035b4aa3a2f4ce44ef291830, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/516b6938ff124f91bc0c8802baa7a75f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/5476ba30c8db4f3eb64e6d7bccd6e784] to archive 2024-11-19T04:42:01,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:42:01,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/330f74150b84412783f5a40240134275 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/330f74150b84412783f5a40240134275 2024-11-19T04:42:01,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/b997f9c92a1048e0b64a5906dcb2b01c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/b997f9c92a1048e0b64a5906dcb2b01c 2024-11-19T04:42:01,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/98c1831bba9a4a89a92376508e32012e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/98c1831bba9a4a89a92376508e32012e 2024-11-19T04:42:01,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/718476832b614784872fb97b31580c18 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/718476832b614784872fb97b31580c18 2024-11-19T04:42:01,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/20252e11b1a148feaf42d365f8846e19 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/20252e11b1a148feaf42d365f8846e19 2024-11-19T04:42:01,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/6894afe7829b4f80b248e8f1c481d4a7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/6894afe7829b4f80b248e8f1c481d4a7 2024-11-19T04:42:01,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/37483eb4a4bb4aca9a09f3a4680e2882 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/37483eb4a4bb4aca9a09f3a4680e2882 2024-11-19T04:42:01,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a5f686b98a6646f5b13ca2989eeee919 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a5f686b98a6646f5b13ca2989eeee919 2024-11-19T04:42:01,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/490a0259774348f883892e3c00e0b470 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/490a0259774348f883892e3c00e0b470 2024-11-19T04:42:01,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/96a5534fe29a48ed805640843f24bd93 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/96a5534fe29a48ed805640843f24bd93 2024-11-19T04:42:01,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/26b4fba33da4480bb8a2e547b1d1d0a5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/26b4fba33da4480bb8a2e547b1d1d0a5 2024-11-19T04:42:01,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/e5f1141754ec4089a1761f805ef9f412 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/e5f1141754ec4089a1761f805ef9f412 2024-11-19T04:42:01,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ceffa79a7cca40ef82967a0126141507 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ceffa79a7cca40ef82967a0126141507 2024-11-19T04:42:01,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca0cd7d27d4e4692aeaa2f606644b9d0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca0cd7d27d4e4692aeaa2f606644b9d0 2024-11-19T04:42:01,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1b7de0a7721d4e00a03a2c78d57a1547 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1b7de0a7721d4e00a03a2c78d57a1547 2024-11-19T04:42:01,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dbaf685abd9e429bb6bead1916df8f12 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dbaf685abd9e429bb6bead1916df8f12 2024-11-19T04:42:01,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/2222ab68110645b09cf793e492764461 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/2222ab68110645b09cf793e492764461 2024-11-19T04:42:01,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a427d903985c4f8f883a71e23914289a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a427d903985c4f8f883a71e23914289a 2024-11-19T04:42:01,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/9ff94c0b24a74b0dbb5420e76a00aea3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/9ff94c0b24a74b0dbb5420e76a00aea3 2024-11-19T04:42:01,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/de06f54c9ae7466bbe2e518d61da4011 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/de06f54c9ae7466bbe2e518d61da4011 2024-11-19T04:42:01,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bf2abd1902974bd5903b8255137391ad to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bf2abd1902974bd5903b8255137391ad 2024-11-19T04:42:01,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/61bdb482af4943b1b9d310ef168999f6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/61bdb482af4943b1b9d310ef168999f6 2024-11-19T04:42:01,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/14547592efeb409180e9a3dea4d6317b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/14547592efeb409180e9a3dea4d6317b 2024-11-19T04:42:01,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca908d9a3fd649399b7b17897606cfe0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/ca908d9a3fd649399b7b17897606cfe0 2024-11-19T04:42:01,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1db72321afab4fca96c29d12b6cbd220 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/1db72321afab4fca96c29d12b6cbd220 2024-11-19T04:42:01,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dfa1e0685c164050828ec9ea392c0406 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/dfa1e0685c164050828ec9ea392c0406 2024-11-19T04:42:01,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/95543d1b51aa4e6081fa8c42444a4bae to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/95543d1b51aa4e6081fa8c42444a4bae 2024-11-19T04:42:01,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/c1517d079b354c9080b459e4462ef067 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/c1517d079b354c9080b459e4462ef067 2024-11-19T04:42:01,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bdbc93b5259a44e3a056b459acdec6d5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/bdbc93b5259a44e3a056b459acdec6d5 2024-11-19T04:42:01,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/fbe9b57d035b4aa3a2f4ce44ef291830 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/fbe9b57d035b4aa3a2f4ce44ef291830 2024-11-19T04:42:01,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/516b6938ff124f91bc0c8802baa7a75f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/516b6938ff124f91bc0c8802baa7a75f 2024-11-19T04:42:01,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/5476ba30c8db4f3eb64e6d7bccd6e784 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/5476ba30c8db4f3eb64e6d7bccd6e784 2024-11-19T04:42:01,733 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/recovered.edits/789.seqid, newMaxSeqId=789, maxSeqId=1 2024-11-19T04:42:01,733 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. 2024-11-19T04:42:01,733 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 641e0316dd7be3ea1908e5148ffce278: Waiting for close lock at 1731991320790Running coprocessor pre-close hooks at 1731991320790Disabling compacts and flushes for region at 1731991320790Disabling writes for close at 1731991320790Obtaining lock to block concurrent updates at 1731991320790Preparing flush snapshotting stores in 641e0316dd7be3ea1908e5148ffce278 at 1731991320790Finished memstore snapshotting TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731991320791 (+1 ms)Flushing stores of TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278. at 1731991320791Flushing 641e0316dd7be3ea1908e5148ffce278/A: creating writer at 1731991320791Flushing 641e0316dd7be3ea1908e5148ffce278/A: appending metadata at 1731991320794 (+3 ms)Flushing 641e0316dd7be3ea1908e5148ffce278/A: closing flushed file at 1731991320794Flushing 641e0316dd7be3ea1908e5148ffce278/B: creating writer at 1731991321201 (+407 ms)Flushing 641e0316dd7be3ea1908e5148ffce278/B: appending metadata at 1731991321203 (+2 ms)Flushing 641e0316dd7be3ea1908e5148ffce278/B: closing flushed file at 1731991321203Flushing 641e0316dd7be3ea1908e5148ffce278/C: creating writer at 1731991321210 (+7 ms)Flushing 641e0316dd7be3ea1908e5148ffce278/C: appending metadata at 1731991321212 (+2 ms)Flushing 641e0316dd7be3ea1908e5148ffce278/C: closing flushed file at 1731991321212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fc758b5: reopening flushed file at 1731991321619 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ef5cbc6: reopening flushed file at 1731991321623 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66eb6189: reopening flushed file at 1731991321626 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 641e0316dd7be3ea1908e5148ffce278 in 841ms, sequenceid=786, compaction requested=false at 1731991321631 (+5 ms)Writing region close event to WAL at 1731991321730 (+99 ms)Running coprocessor post-close hooks at 1731991321733 (+3 ms)Closed at 1731991321733 2024-11-19T04:42:01,735 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:42:01,735 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=641e0316dd7be3ea1908e5148ffce278, regionState=CLOSED 2024-11-19T04:42:01,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:42:01,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-19T04:42:01,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 641e0316dd7be3ea1908e5148ffce278, server=08a7f35e60d4,32975,1731991233019 in 1.0990 sec 2024-11-19T04:42:01,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T04:42:01,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=641e0316dd7be3ea1908e5148ffce278, UNASSIGN in 1.1040 sec 2024-11-19T04:42:01,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-19T04:42:01,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1060 sec 2024-11-19T04:42:01,743 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991321743"}]},"ts":"1731991321743"} 2024-11-19T04:42:01,744 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T04:42:01,744 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T04:42:01,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1150 sec 2024-11-19T04:42:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-19T04:42:01,767 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:01,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T04:42:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,769 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T04:42:01,770 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,772 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:42:01,774 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/recovered.edits] 2024-11-19T04:42:01,776 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/028d38b336d64a76a854d5c8763b4162 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/028d38b336d64a76a854d5c8763b4162 2024-11-19T04:42:01,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/10084accd117421c939ef15adc22bcce to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/A/10084accd117421c939ef15adc22bcce 2024-11-19T04:42:01,778 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/26b1c7b0edb44b71a68c81a95b820bf3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/26b1c7b0edb44b71a68c81a95b820bf3 2024-11-19T04:42:01,779 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/7840769b3160497aaa83cd39c38eeb8b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/B/7840769b3160497aaa83cd39c38eeb8b 2024-11-19T04:42:01,781 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/7e1111355d18473b847cf72534b59bdb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/7e1111355d18473b847cf72534b59bdb 2024-11-19T04:42:01,782 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a3d82e6edae34eef9c80750392f9ab31 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/C/a3d82e6edae34eef9c80750392f9ab31 2024-11-19T04:42:01,785 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/recovered.edits/789.seqid to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278/recovered.edits/789.seqid 2024-11-19T04:42:01,785 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/641e0316dd7be3ea1908e5148ffce278 2024-11-19T04:42:01,785 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T04:42:01,787 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,790 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T04:42:01,792 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T04:42:01,793 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,793 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T04:42:01,793 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731991321793"}]},"ts":"9223372036854775807"} 2024-11-19T04:42:01,795 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T04:42:01,795 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 641e0316dd7be3ea1908e5148ffce278, NAME => 'TestAcidGuarantees,,1731991291428.641e0316dd7be3ea1908e5148ffce278.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T04:42:01,795 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T04:42:01,795 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731991321795"}]},"ts":"9223372036854775807"} 2024-11-19T04:42:01,797 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T04:42:01,797 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-19T04:42:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-19T04:42:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T04:42:01,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:01,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=455 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=512 (was 505) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12109 (was 12457) 2024-11-19T04:42:01,895 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=512, ProcessCount=11, AvailableMemoryMB=12108 2024-11-19T04:42:01,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:42:01,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:42:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:01,898 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T04:42:01,898 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:01,899 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T04:42:01,899 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-19T04:42:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T04:42:01,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742194_1370 (size=1061) 2024-11-19T04:42:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T04:42:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T04:42:02,305 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:42:02,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742195_1371 (size=53) 2024-11-19T04:42:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T04:42:02,657 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T04:42:02,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:42:02,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7cef285b884a1aa14f0a62fe9ee2e32e, disabling compactions & flushes 2024-11-19T04:42:02,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:02,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:02,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. after waiting 0 ms 2024-11-19T04:42:02,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:02,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:02,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7cef285b884a1aa14f0a62fe9ee2e32e: Waiting for close lock at 1731991322711Disabling compacts and flushes for region at 1731991322711Disabling writes for close at 1731991322711Writing region close event to WAL at 1731991322711Closed at 1731991322711 2024-11-19T04:42:02,712 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T04:42:02,712 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731991322712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731991322712"}]},"ts":"1731991322712"} 2024-11-19T04:42:02,714 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T04:42:02,714 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T04:42:02,715 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991322714"}]},"ts":"1731991322714"} 2024-11-19T04:42:02,716 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T04:42:02,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, ASSIGN}] 2024-11-19T04:42:02,717 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, ASSIGN 2024-11-19T04:42:02,718 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, ASSIGN; state=OFFLINE, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=false 2024-11-19T04:42:02,868 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:02,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, ASSIGN because future has completed 2024-11-19T04:42:02,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:42:03,024 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:03,025 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:42:03,025 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,025 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:42:03,025 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,025 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,026 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,027 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:03,027 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cef285b884a1aa14f0a62fe9ee2e32e columnFamilyName A 2024-11-19T04:42:03,027 DEBUG [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:03,028 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(327): Store=7cef285b884a1aa14f0a62fe9ee2e32e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:03,028 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,029 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:03,029 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cef285b884a1aa14f0a62fe9ee2e32e columnFamilyName B 2024-11-19T04:42:03,029 DEBUG [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:03,029 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(327): Store=7cef285b884a1aa14f0a62fe9ee2e32e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:03,029 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,030 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:03,030 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cef285b884a1aa14f0a62fe9ee2e32e columnFamilyName C 2024-11-19T04:42:03,030 DEBUG [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:03,031 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(327): Store=7cef285b884a1aa14f0a62fe9ee2e32e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:03,031 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:03,031 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,031 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,032 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,032 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,032 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,033 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:42:03,034 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,035 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:42:03,036 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 7cef285b884a1aa14f0a62fe9ee2e32e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67500246, jitterRate=0.005832046270370483}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:42:03,036 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:03,036 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 7cef285b884a1aa14f0a62fe9ee2e32e: Running coprocessor pre-open hook at 1731991323025Writing region info on filesystem at 1731991323025Initializing all the Stores at 1731991323026 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991323026Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991323026Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991323026Cleaning up temporary data from old regions at 1731991323032 (+6 ms)Running coprocessor post-open hooks at 1731991323036 (+4 ms)Region opened successfully at 1731991323036 2024-11-19T04:42:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T04:42:03,037 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., pid=99, masterSystemTime=1731991323022 2024-11-19T04:42:03,038 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:03,038 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:03,039 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:03,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:42:03,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-19T04:42:03,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 in 171 msec 2024-11-19T04:42:03,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T04:42:03,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, ASSIGN in 326 msec 2024-11-19T04:42:03,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T04:42:03,044 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991323044"}]},"ts":"1731991323044"} 2024-11-19T04:42:03,045 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T04:42:03,046 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T04:42:03,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-19T04:42:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T04:42:04,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:04,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,059 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b5a2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,061 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279427f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:04,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:04,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:04,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:04,064 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:42:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:42:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ff7e310 2024-11-19T04:42:04,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:42:04,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:42:04,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:42:04,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:42:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742196_1372 (size=1097) 2024-11-19T04:42:04,478 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T04:42:04,478 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T04:42:04,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:42:04,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, REOPEN/MOVE}] 2024-11-19T04:42:04,483 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, REOPEN/MOVE 2024-11-19T04:42:04,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:04,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, REOPEN/MOVE because future has completed 2024-11-19T04:42:04,485 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:42:04,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:42:04,637 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,637 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:42:04,637 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 7cef285b884a1aa14f0a62fe9ee2e32e, disabling compactions & flushes 2024-11-19T04:42:04,637 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,637 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,638 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. after waiting 0 ms 2024-11-19T04:42:04,638 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,641 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T04:42:04,641 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,642 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 7cef285b884a1aa14f0a62fe9ee2e32e: Waiting for close lock at 1731991324637Running coprocessor pre-close hooks at 1731991324637Disabling compacts and flushes for region at 1731991324637Disabling writes for close at 1731991324638 (+1 ms)Writing region close event to WAL at 1731991324638Running coprocessor post-close hooks at 1731991324641 (+3 ms)Closed at 1731991324641 2024-11-19T04:42:04,642 WARN [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 7cef285b884a1aa14f0a62fe9ee2e32e to self. 2024-11-19T04:42:04,643 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,644 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=CLOSED 2024-11-19T04:42:04,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:42:04,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-19T04:42:04,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 in 161 msec 2024-11-19T04:42:04,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, REOPEN/MOVE; state=CLOSED, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=true 2024-11-19T04:42:04,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:04,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, REOPEN/MOVE because future has completed 2024-11-19T04:42:04,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:42:04,954 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,955 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:42:04,955 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,955 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:42:04,955 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,955 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,956 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,957 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:04,957 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cef285b884a1aa14f0a62fe9ee2e32e columnFamilyName A 2024-11-19T04:42:04,958 DEBUG [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:04,958 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(327): Store=7cef285b884a1aa14f0a62fe9ee2e32e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:04,959 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,959 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:04,959 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cef285b884a1aa14f0a62fe9ee2e32e columnFamilyName B 2024-11-19T04:42:04,959 DEBUG [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:04,960 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(327): Store=7cef285b884a1aa14f0a62fe9ee2e32e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:04,960 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,960 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:04,960 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cef285b884a1aa14f0a62fe9ee2e32e columnFamilyName C 2024-11-19T04:42:04,960 DEBUG [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:04,961 INFO [StoreOpener-7cef285b884a1aa14f0a62fe9ee2e32e-1 {}] regionserver.HStore(327): Store=7cef285b884a1aa14f0a62fe9ee2e32e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:04,961 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,961 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,961 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,962 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,962 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,962 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,963 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:42:04,963 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,964 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 7cef285b884a1aa14f0a62fe9ee2e32e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69033005, jitterRate=0.028671935200691223}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:42:04,964 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:04,965 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 7cef285b884a1aa14f0a62fe9ee2e32e: Running coprocessor pre-open hook at 1731991324955Writing region info on filesystem at 1731991324955Initializing all the Stores at 1731991324956 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991324956Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991324956Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991324956Cleaning up temporary data from old regions at 1731991324962 (+6 ms)Running coprocessor post-open hooks at 1731991324964 (+2 ms)Region opened successfully at 1731991324965 (+1 ms) 2024-11-19T04:42:04,965 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., pid=104, masterSystemTime=1731991324952 2024-11-19T04:42:04,967 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=OPEN, openSeqNum=5, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:04,968 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,968 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:04,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:42:04,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-19T04:42:04,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 in 169 msec 2024-11-19T04:42:04,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T04:42:04,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, REOPEN/MOVE in 488 msec 2024-11-19T04:42:04,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-19T04:42:04,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-19T04:42:04,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-19T04:42:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-19T04:42:04,977 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:04,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,978 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,980 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eff2a16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,981 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bab07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17954767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,984 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c2b751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,985 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4015ca78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,986 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd9a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,988 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6ab695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,989 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d3c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6a1a26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,992 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56aca6f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,993 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a77da0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,994 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482db613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,996 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50683acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:04,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ae6359, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:04,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ea3819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:04,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:04,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:05,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6047d1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:05,001 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194c67ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:05,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:05,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:05,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:05,002 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa7cda7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:05,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709d729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:05,005 DEBUG [Thread-1674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,005 DEBUG [Thread-1678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,005 DEBUG [Thread-1693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,005 DEBUG [Thread-1687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,005 DEBUG [Thread-1691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,006 DEBUG [Thread-1682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,007 DEBUG [Thread-1685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,007 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-19T04:42:05,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:05,009 DEBUG [Thread-1676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,011 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:05,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:05,011 DEBUG [Thread-1689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,017 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:05,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:42:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:05,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:05,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:05,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:05,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111971ede7ba25714fac9c034714ce5b1151_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991325027/Put/seqid=0 2024-11-19T04:42:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742197_1373 (size=17034) 2024-11-19T04:42:05,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991325037, max timestamp=1731991325066Num uniques 859; 2024-11-19T04:42:05,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:05,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991325037, max timestamp=1731991325066Num uniques 859; 2024-11-19T04:42:05,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:05,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991325037, max timestamp=1731991325066Num uniques 859; 2024-11-19T04:42:05,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:05,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:05,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111971ede7ba25714fac9c034714ce5b1151_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111971ede7ba25714fac9c034714ce5b1151_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:05,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/44272e64efb241949a36f683766e402f, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:05,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/44272e64efb241949a36f683766e402f is 175, key is test_row_0/A:col10/1731991325027/Put/seqid=0 2024-11-19T04:42:05,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742198_1374 (size=48139) 2024-11-19T04:42:05,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/44272e64efb241949a36f683766e402f 2024-11-19T04:42:05,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/3e15a865afcd472da0d878d1d727e530 is 50, key is test_row_0/B:col10/1731991325027/Put/seqid=0 2024-11-19T04:42:05,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742199_1375 (size=12001) 2024-11-19T04:42:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991385097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991385098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991385106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991385108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991385109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991385217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991385217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991385227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991385227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991385227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991385427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991385437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991385447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991385447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991385447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/3e15a865afcd472da0d878d1d727e530 2024-11-19T04:42:05,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/6d5e43f5e97e4b199d2fa5954d91d32c is 50, key is test_row_0/C:col10/1731991325027/Put/seqid=0 2024-11-19T04:42:05,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742200_1376 (size=12001) 2024-11-19T04:42:05,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:05,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991385747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991385757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991385767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991385767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991385767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:05,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:05,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:05,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/6d5e43f5e97e4b199d2fa5954d91d32c 2024-11-19T04:42:05,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/44272e64efb241949a36f683766e402f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f 2024-11-19T04:42:05,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f, entries=250, sequenceid=15, filesize=47.0 K 2024-11-19T04:42:05,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/3e15a865afcd472da0d878d1d727e530 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/3e15a865afcd472da0d878d1d727e530 2024-11-19T04:42:05,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/3e15a865afcd472da0d878d1d727e530, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:42:05,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/6d5e43f5e97e4b199d2fa5954d91d32c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/6d5e43f5e97e4b199d2fa5954d91d32c 2024-11-19T04:42:05,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/6d5e43f5e97e4b199d2fa5954d91d32c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-19T04:42:05,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7cef285b884a1aa14f0a62fe9ee2e32e in 933ms, sequenceid=15, compaction requested=false 2024-11-19T04:42:05,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:06,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:06,081 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:06,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:06,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191d43ee99d01748d5bc5de22ad3ecf7f5_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991325105/Put/seqid=0 2024-11-19T04:42:06,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742201_1377 (size=12154) 2024-11-19T04:42:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:06,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991386290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991386291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991386298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991386300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991386301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991386407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991386407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991386417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991386418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991386427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:06,498 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191d43ee99d01748d5bc5de22ad3ecf7f5_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191d43ee99d01748d5bc5de22ad3ecf7f5_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:06,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/bbcb9a06d8b34061b8532899a97553cc, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/bbcb9a06d8b34061b8532899a97553cc is 175, key is test_row_0/A:col10/1731991325105/Put/seqid=0 2024-11-19T04:42:06,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742202_1378 (size=30955) 2024-11-19T04:42:06,503 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/bbcb9a06d8b34061b8532899a97553cc 2024-11-19T04:42:06,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/8a0f8825f6774001a9d2170fe81aac38 is 50, key is test_row_0/B:col10/1731991325105/Put/seqid=0 2024-11-19T04:42:06,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742203_1379 (size=12001) 2024-11-19T04:42:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991386627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991386628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991386637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991386637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991386637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,914 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/8a0f8825f6774001a9d2170fe81aac38 2024-11-19T04:42:06,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e4f22d73479d4185bc9a35b24803c832 is 50, key is test_row_0/C:col10/1731991325105/Put/seqid=0 2024-11-19T04:42:06,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742204_1380 (size=12001) 2024-11-19T04:42:06,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991386937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991386947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991386957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991386957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991386957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:07,256 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T04:42:07,333 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e4f22d73479d4185bc9a35b24803c832 2024-11-19T04:42:07,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/bbcb9a06d8b34061b8532899a97553cc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc 2024-11-19T04:42:07,341 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc, entries=150, sequenceid=50, filesize=30.2 K 2024-11-19T04:42:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/8a0f8825f6774001a9d2170fe81aac38 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/8a0f8825f6774001a9d2170fe81aac38 2024-11-19T04:42:07,345 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/8a0f8825f6774001a9d2170fe81aac38, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T04:42:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e4f22d73479d4185bc9a35b24803c832 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e4f22d73479d4185bc9a35b24803c832 2024-11-19T04:42:07,349 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e4f22d73479d4185bc9a35b24803c832, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T04:42:07,350 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1269ms, sequenceid=50, compaction requested=false 2024-11-19T04:42:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-19T04:42:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-19T04:42:07,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T04:42:07,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3410 sec 2024-11-19T04:42:07,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3470 sec 2024-11-19T04:42:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:07,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:42:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:07,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119624a95c0b195498e8529670536e90db1_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:07,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742205_1381 (size=14594) 2024-11-19T04:42:07,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991327497, max timestamp=1731991327510Num uniques 859; 2024-11-19T04:42:07,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991327497, max timestamp=1731991327510Num uniques 859; 2024-11-19T04:42:07,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991327497, max timestamp=1731991327510Num uniques 859; 2024-11-19T04:42:07,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:07,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:07,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:07,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991387538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991387549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991387550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991387550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991387552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991387657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991387657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991387667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991387677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991387677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:07,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119624a95c0b195498e8529670536e90db1_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119624a95c0b195498e8529670536e90db1_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:07,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/db5630e5246a4991876b7a118214c120, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:07,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/db5630e5246a4991876b7a118214c120 is 175, key is test_row_0/A:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742206_1382 (size=39549) 2024-11-19T04:42:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991387877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991387877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991387887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991387896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991387896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991388197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991388197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991388207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991388207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991388207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/db5630e5246a4991876b7a118214c120 2024-11-19T04:42:08,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d18bed6cb4234ac7a9746941c0c4a14b is 50, key is test_row_0/B:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:08,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742207_1383 (size=12001) 2024-11-19T04:42:08,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d18bed6cb4234ac7a9746941c0c4a14b 2024-11-19T04:42:08,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/17bd0c97caaf438888b9940e0b812e74 is 50, key is test_row_0/C:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:08,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742208_1384 (size=12001) 2024-11-19T04:42:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991388716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991388717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991388727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991388727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991388727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/17bd0c97caaf438888b9940e0b812e74 2024-11-19T04:42:09,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/db5630e5246a4991876b7a118214c120 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120 2024-11-19T04:42:09,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120, entries=200, sequenceid=62, filesize=38.6 K 2024-11-19T04:42:09,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d18bed6cb4234ac7a9746941c0c4a14b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d18bed6cb4234ac7a9746941c0c4a14b 2024-11-19T04:42:09,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d18bed6cb4234ac7a9746941c0c4a14b, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T04:42:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/17bd0c97caaf438888b9940e0b812e74 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/17bd0c97caaf438888b9940e0b812e74 2024-11-19T04:42:09,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/17bd0c97caaf438888b9940e0b812e74, entries=150, sequenceid=62, filesize=11.7 K 2024-11-19T04:42:09,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1668ms, sequenceid=62, compaction requested=true 2024-11-19T04:42:09,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:09,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:09,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:09,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:09,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:09,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:09,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/A is initiating minor compaction (all files) 2024-11-19T04:42:09,121 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/A in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/B is initiating minor compaction (all files) 2024-11-19T04:42:09,121 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/B in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:09,121 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=115.9 K 2024-11-19T04:42:09,121 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:09,121 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120] 2024-11-19T04:42:09,122 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/3e15a865afcd472da0d878d1d727e530, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/8a0f8825f6774001a9d2170fe81aac38, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d18bed6cb4234ac7a9746941c0c4a14b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=35.2 K 2024-11-19T04:42:09,122 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44272e64efb241949a36f683766e402f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991325012 2024-11-19T04:42:09,122 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e15a865afcd472da0d878d1d727e530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991325023 2024-11-19T04:42:09,122 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbcb9a06d8b34061b8532899a97553cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731991325103 2024-11-19T04:42:09,122 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0f8825f6774001a9d2170fe81aac38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731991325103 2024-11-19T04:42:09,122 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5630e5246a4991876b7a118214c120, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731991326276 2024-11-19T04:42:09,122 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d18bed6cb4234ac7a9746941c0c4a14b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731991326298 2024-11-19T04:42:09,127 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:09,129 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#B#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:09,129 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/23f59f025d554806823c16bb842700b7 is 50, key is test_row_0/B:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:09,129 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194377b0892031463f87038edbf41ac7c3_7cef285b884a1aa14f0a62fe9ee2e32e store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:09,132 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194377b0892031463f87038edbf41ac7c3_7cef285b884a1aa14f0a62fe9ee2e32e, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:09,132 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194377b0892031463f87038edbf41ac7c3_7cef285b884a1aa14f0a62fe9ee2e32e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742209_1385 (size=4469) 2024-11-19T04:42:09,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742210_1386 (size=12104) 2024-11-19T04:42:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T04:42:09,187 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:09,187 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-19T04:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T04:42:09,189 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:09,190 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:09,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T04:42:09,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-19T04:42:09,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:09,342 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-19T04:42:09,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:09,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:09,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:09,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:09,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:09,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:09,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ae1a5466585344f594c910d66c81728a_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991327532/Put/seqid=0 2024-11-19T04:42:09,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742211_1387 (size=12154) 2024-11-19T04:42:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T04:42:09,541 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:09,542 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/5e447185343a4ca9a17a2474b7667a0d is 175, key is test_row_0/A:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:09,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742212_1388 (size=31058) 2024-11-19T04:42:09,548 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/23f59f025d554806823c16bb842700b7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/23f59f025d554806823c16bb842700b7 2024-11-19T04:42:09,558 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/B of 7cef285b884a1aa14f0a62fe9ee2e32e into 23f59f025d554806823c16bb842700b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:09,558 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:09,558 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/B, priority=13, startTime=1731991329120; duration=0sec 2024-11-19T04:42:09,558 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:09,558 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:B 2024-11-19T04:42:09,558 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:09,559 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:09,559 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/C is initiating minor compaction (all files) 2024-11-19T04:42:09,559 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/C in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:09,559 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/6d5e43f5e97e4b199d2fa5954d91d32c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e4f22d73479d4185bc9a35b24803c832, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/17bd0c97caaf438888b9940e0b812e74] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=35.2 K 2024-11-19T04:42:09,559 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5e43f5e97e4b199d2fa5954d91d32c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991325023 2024-11-19T04:42:09,560 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f22d73479d4185bc9a35b24803c832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731991325103 2024-11-19T04:42:09,560 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 17bd0c97caaf438888b9940e0b812e74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731991326298 2024-11-19T04:42:09,566 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#C#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:09,566 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/bdf9c74df8ca4c2d8f917a78c72eb166 is 50, key is test_row_0/C:col10/1731991327451/Put/seqid=0 2024-11-19T04:42:09,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742213_1389 (size=12104) 2024-11-19T04:42:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:09,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:09,759 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ae1a5466585344f594c910d66c81728a_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ae1a5466585344f594c910d66c81728a_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/f368fc9eaca94e06802eb4d9a7880476, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:09,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/f368fc9eaca94e06802eb4d9a7880476 is 175, key is test_row_0/A:col10/1731991327532/Put/seqid=0 2024-11-19T04:42:09,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742214_1390 (size=30955) 2024-11-19T04:42:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991389758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991389760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991389761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991389763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991389767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T04:42:09,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991389876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991389887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991389887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991389887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991389897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:09,950 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/5e447185343a4ca9a17a2474b7667a0d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/5e447185343a4ca9a17a2474b7667a0d 2024-11-19T04:42:09,954 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/A of 7cef285b884a1aa14f0a62fe9ee2e32e into 5e447185343a4ca9a17a2474b7667a0d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:09,954 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:09,954 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/A, priority=13, startTime=1731991329120; duration=0sec 2024-11-19T04:42:09,954 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:09,954 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:A 2024-11-19T04:42:09,973 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/bdf9c74df8ca4c2d8f917a78c72eb166 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/bdf9c74df8ca4c2d8f917a78c72eb166 2024-11-19T04:42:09,979 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/C of 7cef285b884a1aa14f0a62fe9ee2e32e into bdf9c74df8ca4c2d8f917a78c72eb166(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:09,979 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:09,979 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/C, priority=13, startTime=1731991329121; duration=0sec 2024-11-19T04:42:09,979 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:09,979 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:C 2024-11-19T04:42:10,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991390097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991390107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991390107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991390107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991390107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,165 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/f368fc9eaca94e06802eb4d9a7880476 2024-11-19T04:42:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/7b06690b1d2f47cbb6b42075ff42a786 is 50, key is test_row_0/B:col10/1731991327532/Put/seqid=0 2024-11-19T04:42:10,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742215_1391 (size=12001) 2024-11-19T04:42:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T04:42:10,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991390407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991390427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991390427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991390427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991390427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,576 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/7b06690b1d2f47cbb6b42075ff42a786 2024-11-19T04:42:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/d4b8fdc086d24a69b66d045b6978bab6 is 50, key is test_row_0/C:col10/1731991327532/Put/seqid=0 2024-11-19T04:42:10,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742216_1392 (size=12001) 2024-11-19T04:42:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991390927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991390937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991390937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991390947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991390947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:10,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:10,987 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/d4b8fdc086d24a69b66d045b6978bab6 2024-11-19T04:42:10,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/f368fc9eaca94e06802eb4d9a7880476 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476 2024-11-19T04:42:10,995 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476, entries=150, sequenceid=97, filesize=30.2 K 2024-11-19T04:42:10,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/7b06690b1d2f47cbb6b42075ff42a786 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/7b06690b1d2f47cbb6b42075ff42a786 2024-11-19T04:42:11,005 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/7b06690b1d2f47cbb6b42075ff42a786, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T04:42:11,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/d4b8fdc086d24a69b66d045b6978bab6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d4b8fdc086d24a69b66d045b6978bab6 2024-11-19T04:42:11,010 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d4b8fdc086d24a69b66d045b6978bab6, entries=150, sequenceid=97, filesize=11.7 K 2024-11-19T04:42:11,011 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1669ms, sequenceid=97, compaction requested=false 2024-11-19T04:42:11,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:11,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:11,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-19T04:42:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-19T04:42:11,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T04:42:11,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8220 sec 2024-11-19T04:42:11,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8270 sec 2024-11-19T04:42:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-19T04:42:11,337 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:11,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-19T04:42:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T04:42:11,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:11,340 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:11,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T04:42:11,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-19T04:42:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:11,493 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:42:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:11,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:11,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119686678dea34944c38250083b90594cd6_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991329758/Put/seqid=0 2024-11-19T04:42:11,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742217_1393 (size=12154) 2024-11-19T04:42:11,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:11,544 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119686678dea34944c38250083b90594cd6_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119686678dea34944c38250083b90594cd6_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/acf85a9a1c2d456abff818a5ac389704, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/acf85a9a1c2d456abff818a5ac389704 is 175, key is test_row_0/A:col10/1731991329758/Put/seqid=0 2024-11-19T04:42:11,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742218_1394 (size=30955) 2024-11-19T04:42:11,568 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/acf85a9a1c2d456abff818a5ac389704 2024-11-19T04:42:11,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/b51ac2d42cca41eebeb35502e8736dc6 is 50, key is test_row_0/B:col10/1731991329758/Put/seqid=0 2024-11-19T04:42:11,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742219_1395 (size=12001) 2024-11-19T04:42:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T04:42:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:11,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T04:42:11,992 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/b51ac2d42cca41eebeb35502e8736dc6 2024-11-19T04:42:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/8e53ac18009d41968d32168d74efa953 is 50, key is test_row_0/C:col10/1731991329758/Put/seqid=0 2024-11-19T04:42:12,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742220_1396 (size=12001) 2024-11-19T04:42:12,029 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/8e53ac18009d41968d32168d74efa953 2024-11-19T04:42:12,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/acf85a9a1c2d456abff818a5ac389704 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704 2024-11-19T04:42:12,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731991332006, max timestamp=1731991332036Num uniques 859; 2024-11-19T04:42:12,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:12,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731991332006, max timestamp=1731991332036Num uniques 859; 2024-11-19T04:42:12,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731991332006, max timestamp=1731991332036Num uniques 859; 2024-11-19T04:42:12,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:12,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:12,047 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704, entries=150, sequenceid=112, filesize=30.2 K 2024-11-19T04:42:12,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/b51ac2d42cca41eebeb35502e8736dc6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/b51ac2d42cca41eebeb35502e8736dc6 2024-11-19T04:42:12,054 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/b51ac2d42cca41eebeb35502e8736dc6, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T04:42:12,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/8e53ac18009d41968d32168d74efa953 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/8e53ac18009d41968d32168d74efa953 2024-11-19T04:42:12,059 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/8e53ac18009d41968d32168d74efa953, entries=150, sequenceid=112, filesize=11.7 K 2024-11-19T04:42:12,061 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=53.67 KB/54960 for 7cef285b884a1aa14f0a62fe9ee2e32e in 568ms, sequenceid=112, compaction requested=true 2024-11-19T04:42:12,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:12,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-19T04:42:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-19T04:42:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:12,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=60.38 KB heapSize=161.16 KB 2024-11-19T04:42:12,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:12,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-19T04:42:12,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 723 msec 2024-11-19T04:42:12,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 732 msec 2024-11-19T04:42:12,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a971df0ae95a41bb88860ad232d67e8e_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:12,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742221_1397 (size=17234) 2024-11-19T04:42:12,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991332117, max timestamp=1731991332155Num uniques 859; 2024-11-19T04:42:12,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:12,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991332117, max timestamp=1731991332155Num uniques 859; 2024-11-19T04:42:12,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:12,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991332117, max timestamp=1731991332155Num uniques 859; 2024-11-19T04:42:12,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:12,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991392222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991392222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991392224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991392225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991392226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991392357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991392357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991392357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991392367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991392367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T04:42:12,478 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:12,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-19T04:42:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:12,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:12,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:12,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:12,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:12,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a971df0ae95a41bb88860ad232d67e8e_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a971df0ae95a41bb88860ad232d67e8e_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:12,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d7e088cd38d64a72b77fe7832941050c, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:12,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d7e088cd38d64a72b77fe7832941050c is 175, key is test_row_0/A:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:12,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742222_1398 (size=48339) 2024-11-19T04:42:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991392577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991392587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991392588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991392588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991392588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,657 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T04:42:12,657 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T04:42:12,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991392896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991392907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991392907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991392907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991392907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:12,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:12,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:12,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d7e088cd38d64a72b77fe7832941050c 2024-11-19T04:42:12,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/c6096c46c7fe493190a925dae4cab785 is 50, key is test_row_0/B:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:12,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742223_1399 (size=12151) 2024-11-19T04:42:13,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:13,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:13,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:13,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:13,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:13,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/c6096c46c7fe493190a925dae4cab785 2024-11-19T04:42:13,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/1252b4d0613c40f3909495df3fb68d27 is 50, key is test_row_0/C:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:13,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:13,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:13,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991393427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991393427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991393428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991393428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:13,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991393430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742224_1400 (size=12151) 2024-11-19T04:42:13,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:13,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:13,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:13,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:13,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:13,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/1252b4d0613c40f3909495df3fb68d27 2024-11-19T04:42:13,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:13,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:13,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:13,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d7e088cd38d64a72b77fe7832941050c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c 2024-11-19T04:42:13,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c, entries=250, sequenceid=137, filesize=47.2 K 2024-11-19T04:42:13,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/c6096c46c7fe493190a925dae4cab785 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/c6096c46c7fe493190a925dae4cab785 2024-11-19T04:42:13,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/c6096c46c7fe493190a925dae4cab785, entries=150, sequenceid=137, filesize=11.9 K 2024-11-19T04:42:13,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/1252b4d0613c40f3909495df3fb68d27 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/1252b4d0613c40f3909495df3fb68d27 2024-11-19T04:42:13,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/1252b4d0613c40f3909495df3fb68d27, entries=150, sequenceid=137, filesize=11.9 K 2024-11-19T04:42:13,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~161.11 KB/164976, currentSize=127.47 KB/130530 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1818ms, sequenceid=137, compaction requested=true 2024-11-19T04:42:13,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:13,884 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:13,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T04:42:13,884 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:13,885 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:13,885 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/B is initiating minor compaction (all files) 2024-11-19T04:42:13,885 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141307 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:13,885 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/B in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,885 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/A is initiating minor compaction (all files) 2024-11-19T04:42:13,885 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/23f59f025d554806823c16bb842700b7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/7b06690b1d2f47cbb6b42075ff42a786, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/b51ac2d42cca41eebeb35502e8736dc6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/c6096c46c7fe493190a925dae4cab785] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=47.1 K 2024-11-19T04:42:13,885 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/A in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,885 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/5e447185343a4ca9a17a2474b7667a0d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=138.0 K 2024-11-19T04:42:13,885 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:13,885 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/5e447185343a4ca9a17a2474b7667a0d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c] 2024-11-19T04:42:13,886 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f59f025d554806823c16bb842700b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731991326298 2024-11-19T04:42:13,886 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e447185343a4ca9a17a2474b7667a0d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731991326298 2024-11-19T04:42:13,886 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b06690b1d2f47cbb6b42075ff42a786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731991327532 2024-11-19T04:42:13,886 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f368fc9eaca94e06802eb4d9a7880476, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731991327532 2024-11-19T04:42:13,887 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51ac2d42cca41eebeb35502e8736dc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731991329758 2024-11-19T04:42:13,887 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting acf85a9a1c2d456abff818a5ac389704, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731991329758 2024-11-19T04:42:13,887 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6096c46c7fe493190a925dae4cab785, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991332036 2024-11-19T04:42:13,887 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e088cd38d64a72b77fe7832941050c, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991332006 2024-11-19T04:42:13,907 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:13,927 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:13,928 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d7532186ab7642e5bdfdc878ceae9ea0 is 50, key is test_row_0/B:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:13,937 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199420913836bf4d9eb8b585362d86c1b0_7cef285b884a1aa14f0a62fe9ee2e32e store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:13,940 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199420913836bf4d9eb8b585362d86c1b0_7cef285b884a1aa14f0a62fe9ee2e32e, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:13,940 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199420913836bf4d9eb8b585362d86c1b0_7cef285b884a1aa14f0a62fe9ee2e32e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:13,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742225_1401 (size=12391) 2024-11-19T04:42:14,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-19T04:42:14,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:14,019 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-11-19T04:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:14,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742226_1402 (size=4469) 2024-11-19T04:42:14,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197d4a92be05aa40e6949d99c794a9ca55_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991332185/Put/seqid=0 2024-11-19T04:42:14,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742227_1403 (size=12304) 2024-11-19T04:42:14,399 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d7532186ab7642e5bdfdc878ceae9ea0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d7532186ab7642e5bdfdc878ceae9ea0 2024-11-19T04:42:14,404 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/B of 7cef285b884a1aa14f0a62fe9ee2e32e into d7532186ab7642e5bdfdc878ceae9ea0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:14,405 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:14,405 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/B, priority=12, startTime=1731991333884; duration=0sec 2024-11-19T04:42:14,405 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:14,405 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:B 2024-11-19T04:42:14,405 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:14,406 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:14,406 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/C is initiating minor compaction (all files) 2024-11-19T04:42:14,406 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/C in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:14,406 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/bdf9c74df8ca4c2d8f917a78c72eb166, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d4b8fdc086d24a69b66d045b6978bab6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/8e53ac18009d41968d32168d74efa953, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/1252b4d0613c40f3909495df3fb68d27] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=47.1 K 2024-11-19T04:42:14,409 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdf9c74df8ca4c2d8f917a78c72eb166, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731991326298 2024-11-19T04:42:14,409 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b8fdc086d24a69b66d045b6978bab6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731991327532 2024-11-19T04:42:14,409 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e53ac18009d41968d32168d74efa953, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731991329758 2024-11-19T04:42:14,410 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1252b4d0613c40f3909495df3fb68d27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991332036 2024-11-19T04:42:14,420 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:14,420 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/f0693d813907463596c15824aeb0ae26 is 50, key is test_row_0/C:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:14,433 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#A#compaction#339 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:14,434 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/ad916a40f1ea4ceca37964cff6dfb503 is 175, key is test_row_0/A:col10/1731991332047/Put/seqid=0 2024-11-19T04:42:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:14,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:14,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742228_1404 (size=12391) 2024-11-19T04:42:14,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742229_1405 (size=31345) 2024-11-19T04:42:14,477 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/f0693d813907463596c15824aeb0ae26 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f0693d813907463596c15824aeb0ae26 2024-11-19T04:42:14,481 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/C of 7cef285b884a1aa14f0a62fe9ee2e32e into f0693d813907463596c15824aeb0ae26(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:14,481 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:14,481 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/C, priority=12, startTime=1731991333884; duration=0sec 2024-11-19T04:42:14,482 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:14,482 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:C 2024-11-19T04:42:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991394468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991394470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:14,510 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197d4a92be05aa40e6949d99c794a9ca55_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197d4a92be05aa40e6949d99c794a9ca55_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/37e4862bdefe4bac8e41207b8a654d80, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:14,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/37e4862bdefe4bac8e41207b8a654d80 is 175, key is test_row_0/A:col10/1731991332185/Put/seqid=0 2024-11-19T04:42:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991394493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991394494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991394496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742230_1406 (size=31105) 2024-11-19T04:42:14,554 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/37e4862bdefe4bac8e41207b8a654d80 2024-11-19T04:42:14,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d869965ee4ca42db8a8f07e8945b5c59 is 50, key is test_row_0/B:col10/1731991332185/Put/seqid=0 2024-11-19T04:42:14,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742231_1407 (size=12151) 2024-11-19T04:42:14,597 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d869965ee4ca42db8a8f07e8945b5c59 2024-11-19T04:42:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/b26bfd695bfc4792ac5b328d182c4aad is 50, key is test_row_0/C:col10/1731991332185/Put/seqid=0 2024-11-19T04:42:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991394607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991394607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:14,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742232_1408 (size=12151) 2024-11-19T04:42:14,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991394627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991394627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991394627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991394827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991394837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991394847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991394847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991394847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:14,881 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/ad916a40f1ea4ceca37964cff6dfb503 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ad916a40f1ea4ceca37964cff6dfb503 2024-11-19T04:42:14,886 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/A of 7cef285b884a1aa14f0a62fe9ee2e32e into ad916a40f1ea4ceca37964cff6dfb503(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-19T04:42:14,886 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:14,886 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/A, priority=12, startTime=1731991333883; duration=1sec 2024-11-19T04:42:14,886 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:14,886 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:A 2024-11-19T04:42:15,029 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/b26bfd695bfc4792ac5b328d182c4aad 2024-11-19T04:42:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/37e4862bdefe4bac8e41207b8a654d80 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80 2024-11-19T04:42:15,037 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80, entries=150, sequenceid=170, filesize=30.4 K 2024-11-19T04:42:15,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/d869965ee4ca42db8a8f07e8945b5c59 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d869965ee4ca42db8a8f07e8945b5c59 2024-11-19T04:42:15,041 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d869965ee4ca42db8a8f07e8945b5c59, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T04:42:15,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/b26bfd695bfc4792ac5b328d182c4aad as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/b26bfd695bfc4792ac5b328d182c4aad 2024-11-19T04:42:15,045 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/b26bfd695bfc4792ac5b328d182c4aad, entries=150, sequenceid=170, filesize=11.9 K 2024-11-19T04:42:15,046 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=60.38 KB/61830 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1027ms, sequenceid=170, compaction requested=false 2024-11-19T04:42:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-19T04:42:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-19T04:42:15,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T04:42:15,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5660 sec 2024-11-19T04:42:15,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.5720 sec 2024-11-19T04:42:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:15,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-19T04:42:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:15,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:15,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eb22b998451c47b397860f369a229fee_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742233_1409 (size=14794) 2024-11-19T04:42:15,154 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:15,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119eb22b998451c47b397860f369a229fee_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eb22b998451c47b397860f369a229fee_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:15,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d66159a3670e46618b4f19c095709542, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:15,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d66159a3670e46618b4f19c095709542 is 175, key is test_row_0/A:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742234_1410 (size=39749) 2024-11-19T04:42:15,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d66159a3670e46618b4f19c095709542 2024-11-19T04:42:15,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/348540674d0d41beae67f1922a208db6 is 50, key is test_row_0/B:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742235_1411 (size=12151) 2024-11-19T04:42:15,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/348540674d0d41beae67f1922a208db6 2024-11-19T04:42:15,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/c8923e5e00934658981ef1da954a6141 is 50, key is test_row_0/C:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742236_1412 (size=12151) 2024-11-19T04:42:15,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/c8923e5e00934658981ef1da954a6141 2024-11-19T04:42:15,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/d66159a3670e46618b4f19c095709542 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542 2024-11-19T04:42:15,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542, entries=200, sequenceid=186, filesize=38.8 K 2024-11-19T04:42:15,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/348540674d0d41beae67f1922a208db6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/348540674d0d41beae67f1922a208db6 2024-11-19T04:42:15,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/348540674d0d41beae67f1922a208db6, entries=150, sequenceid=186, filesize=11.9 K 2024-11-19T04:42:15,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/c8923e5e00934658981ef1da954a6141 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/c8923e5e00934658981ef1da954a6141 2024-11-19T04:42:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991395196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991395197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/c8923e5e00934658981ef1da954a6141, entries=150, sequenceid=186, filesize=11.9 K 2024-11-19T04:42:15,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7cef285b884a1aa14f0a62fe9ee2e32e in 73ms, sequenceid=186, compaction requested=true 2024-11-19T04:42:15,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:15,216 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:15,216 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:15,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:15,216 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:15,217 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/B is initiating minor compaction (all files) 2024-11-19T04:42:15,217 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/B in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:15,217 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d7532186ab7642e5bdfdc878ceae9ea0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d869965ee4ca42db8a8f07e8945b5c59, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/348540674d0d41beae67f1922a208db6] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=35.8 K 2024-11-19T04:42:15,217 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:15,217 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/A is initiating minor compaction (all files) 2024-11-19T04:42:15,217 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/A in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:15,217 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ad916a40f1ea4ceca37964cff6dfb503, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=99.8 K 2024-11-19T04:42:15,217 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:15,217 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ad916a40f1ea4ceca37964cff6dfb503, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542] 2024-11-19T04:42:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:15,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:42:15,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:15,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:15,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:15,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:15,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:15,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:15,219 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d7532186ab7642e5bdfdc878ceae9ea0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991332036 2024-11-19T04:42:15,219 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad916a40f1ea4ceca37964cff6dfb503, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991332036 2024-11-19T04:42:15,220 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d869965ee4ca42db8a8f07e8945b5c59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731991332185 2024-11-19T04:42:15,220 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e4862bdefe4bac8e41207b8a654d80, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731991332185 2024-11-19T04:42:15,220 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 348540674d0d41beae67f1922a208db6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731991334492 2024-11-19T04:42:15,220 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d66159a3670e46618b4f19c095709542, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731991334490 2024-11-19T04:42:15,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195dc62dc28d3e4205b220f359755e1f31_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991335216/Put/seqid=0 2024-11-19T04:42:15,229 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:15,234 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:15,235 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/49e2e90bdb12499d95c52df34ad01e6c is 50, key is test_row_0/B:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,241 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119112f8baab43e40d19b047413f24594e8_7cef285b884a1aa14f0a62fe9ee2e32e store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:15,243 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119112f8baab43e40d19b047413f24594e8_7cef285b884a1aa14f0a62fe9ee2e32e, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:15,243 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119112f8baab43e40d19b047413f24594e8_7cef285b884a1aa14f0a62fe9ee2e32e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:15,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742237_1413 (size=14794) 2024-11-19T04:42:15,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742238_1414 (size=12493) 2024-11-19T04:42:15,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742239_1415 (size=4469) 2024-11-19T04:42:15,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991395248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991395250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991395251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991395317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991395317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991395366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991395366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991395366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991395527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991395527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991395586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991395587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991395587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,647 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:15,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195dc62dc28d3e4205b220f359755e1f31_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195dc62dc28d3e4205b220f359755e1f31_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:15,651 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/49e2e90bdb12499d95c52df34ad01e6c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/49e2e90bdb12499d95c52df34ad01e6c 2024-11-19T04:42:15,651 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/c25cd0e819734bdf922c0613e82fe2ba, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:15,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/c25cd0e819734bdf922c0613e82fe2ba is 175, key is test_row_0/A:col10/1731991335216/Put/seqid=0 2024-11-19T04:42:15,658 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/B of 7cef285b884a1aa14f0a62fe9ee2e32e into 49e2e90bdb12499d95c52df34ad01e6c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:15,658 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:15,658 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/B, priority=13, startTime=1731991335216; duration=0sec 2024-11-19T04:42:15,659 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:15,659 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:B 2024-11-19T04:42:15,659 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:15,660 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#A#compaction#349 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:15,661 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/96f87838819741bc85d296f24e83d405 is 175, key is test_row_0/A:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742240_1416 (size=39749) 2024-11-19T04:42:15,664 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:15,664 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/C is initiating minor compaction (all files) 2024-11-19T04:42:15,664 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/C in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:15,664 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f0693d813907463596c15824aeb0ae26, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/b26bfd695bfc4792ac5b328d182c4aad, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/c8923e5e00934658981ef1da954a6141] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=35.8 K 2024-11-19T04:42:15,664 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f0693d813907463596c15824aeb0ae26, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731991332036 2024-11-19T04:42:15,665 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b26bfd695bfc4792ac5b328d182c4aad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731991332185 2024-11-19T04:42:15,665 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c8923e5e00934658981ef1da954a6141, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731991334492 2024-11-19T04:42:15,675 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:15,676 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/5e51c9f606374ad5a5684d66ddb7c254 is 50, key is test_row_0/C:col10/1731991334492/Put/seqid=0 2024-11-19T04:42:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742241_1417 (size=31447) 2024-11-19T04:42:15,687 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/96f87838819741bc85d296f24e83d405 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/96f87838819741bc85d296f24e83d405 2024-11-19T04:42:15,692 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/A of 7cef285b884a1aa14f0a62fe9ee2e32e into 96f87838819741bc85d296f24e83d405(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:15,692 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:15,692 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/A, priority=13, startTime=1731991335215; duration=0sec 2024-11-19T04:42:15,692 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:15,692 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:A 2024-11-19T04:42:15,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742242_1418 (size=12493) 2024-11-19T04:42:15,703 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/5e51c9f606374ad5a5684d66ddb7c254 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/5e51c9f606374ad5a5684d66ddb7c254 2024-11-19T04:42:15,709 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/C of 7cef285b884a1aa14f0a62fe9ee2e32e into 5e51c9f606374ad5a5684d66ddb7c254(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:15,710 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:15,710 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/C, priority=13, startTime=1731991335216; duration=0sec 2024-11-19T04:42:15,710 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:15,710 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:C 2024-11-19T04:42:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991395837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991395847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991395907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991395907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991395907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:16,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/c25cd0e819734bdf922c0613e82fe2ba 2024-11-19T04:42:16,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/aa482663c8c34b3ab32655915d721d87 is 50, key is test_row_0/B:col10/1731991335216/Put/seqid=0 2024-11-19T04:42:16,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742243_1419 (size=12151) 2024-11-19T04:42:16,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/aa482663c8c34b3ab32655915d721d87 2024-11-19T04:42:16,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/4bfad097272748cd9aa750e67e054296 is 50, key is test_row_0/C:col10/1731991335216/Put/seqid=0 2024-11-19T04:42:16,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742244_1420 (size=12151) 2024-11-19T04:42:16,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991396357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991396367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:16,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991396427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:16,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991396427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:16,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991396427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:16,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:16,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/4bfad097272748cd9aa750e67e054296 2024-11-19T04:42:16,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/c25cd0e819734bdf922c0613e82fe2ba as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba 2024-11-19T04:42:16,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba, entries=200, sequenceid=206, filesize=38.8 K 2024-11-19T04:42:16,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/aa482663c8c34b3ab32655915d721d87 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/aa482663c8c34b3ab32655915d721d87 2024-11-19T04:42:16,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/aa482663c8c34b3ab32655915d721d87, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T04:42:16,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/4bfad097272748cd9aa750e67e054296 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/4bfad097272748cd9aa750e67e054296 2024-11-19T04:42:16,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/4bfad097272748cd9aa750e67e054296, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T04:42:16,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1286ms, sequenceid=206, compaction requested=false 2024-11-19T04:42:16,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-19T04:42:16,637 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:16,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T04:42:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T04:42:16,640 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:16,641 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:16,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T04:42:16,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T04:42:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:42:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119caed8f3ad05043a2b386d54014b3f4ad_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991335249/Put/seqid=0 2024-11-19T04:42:16,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742245_1421 (size=12304) 2024-11-19T04:42:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T04:42:17,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:17,210 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119caed8f3ad05043a2b386d54014b3f4ad_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119caed8f3ad05043a2b386d54014b3f4ad_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:17,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/9e1a78d1e63f473ca1dab049197c9d7f, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:17,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/9e1a78d1e63f473ca1dab049197c9d7f is 175, key is test_row_0/A:col10/1731991335249/Put/seqid=0 2024-11-19T04:42:17,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742246_1422 (size=31105) 2024-11-19T04:42:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T04:42:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:17,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:17,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991397445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991397446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991397447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991397447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991397448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,460 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 7cef285b884a1aa14f0a62fe9ee2e32e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991397557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991397567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991397567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991397567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,617 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/9e1a78d1e63f473ca1dab049197c9d7f 2024-11-19T04:42:17,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/54537f646d514bc289c91e831929ca33 is 50, key is test_row_0/B:col10/1731991335249/Put/seqid=0 2024-11-19T04:42:17,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742247_1423 (size=12151) 2024-11-19T04:42:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T04:42:17,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991397777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991397786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991397787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991397787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,032 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/54537f646d514bc289c91e831929ca33 2024-11-19T04:42:18,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/29d74c4f8432425fa5f2c6a471313c99 is 50, key is test_row_0/C:col10/1731991335249/Put/seqid=0 2024-11-19T04:42:18,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742248_1424 (size=12151) 2024-11-19T04:42:18,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991398097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991398097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991398097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991398106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,442 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/29d74c4f8432425fa5f2c6a471313c99 2024-11-19T04:42:18,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/9e1a78d1e63f473ca1dab049197c9d7f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f 2024-11-19T04:42:18,451 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f, entries=150, sequenceid=221, filesize=30.4 K 2024-11-19T04:42:18,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/54537f646d514bc289c91e831929ca33 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/54537f646d514bc289c91e831929ca33 2024-11-19T04:42:18,455 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/54537f646d514bc289c91e831929ca33, entries=150, sequenceid=221, filesize=11.9 K 2024-11-19T04:42:18,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/29d74c4f8432425fa5f2c6a471313c99 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/29d74c4f8432425fa5f2c6a471313c99 2024-11-19T04:42:18,459 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/29d74c4f8432425fa5f2c6a471313c99, entries=150, sequenceid=221, filesize=11.9 K 2024-11-19T04:42:18,459 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1665ms, sequenceid=221, compaction requested=true 2024-11-19T04:42:18,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:18,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:18,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T04:42:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T04:42:18,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T04:42:18,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-11-19T04:42:18,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8250 sec 2024-11-19T04:42:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:18,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T04:42:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:18,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:18,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e364587e41eb4da29f22b181f7dcdd7b_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:18,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742249_1425 (size=14794) 2024-11-19T04:42:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991398630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991398631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991398637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991398640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991398757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991398757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991398757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991398766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T04:42:18,787 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:18,787 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T04:42:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:18,789 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:18,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:18,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:18,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:18,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:18,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991398967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991398967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991398967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:18,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991398987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:19,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e364587e41eb4da29f22b181f7dcdd7b_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e364587e41eb4da29f22b181f7dcdd7b_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:19,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/80f8ffa5a24d4bac9143087186d95827, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:19,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/80f8ffa5a24d4bac9143087186d95827 is 175, key is test_row_0/A:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:19,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742250_1426 (size=39749) 2024-11-19T04:42:19,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/80f8ffa5a24d4bac9143087186d95827 2024-11-19T04:42:19,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/6adfa6dd0d8e4770ab0053562e7a6c6a is 50, key is test_row_0/B:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:19,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742251_1427 (size=12151) 2024-11-19T04:42:19,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:19,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:19,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:19,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991399287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991399287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991399287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991399296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:19,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/6adfa6dd0d8e4770ab0053562e7a6c6a 2024-11-19T04:42:19,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/ae4feb1754304d14be2a636a7135436c is 50, key is test_row_0/C:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742252_1428 (size=12151) 2024-11-19T04:42:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991399487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,496 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 7cef285b884a1aa14f0a62fe9ee2e32e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:19,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:19,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:19,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991399797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991399797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991399807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991399807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:19,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/ae4feb1754304d14be2a636a7135436c 2024-11-19T04:42:19,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/80f8ffa5a24d4bac9143087186d95827 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827 2024-11-19T04:42:19,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827, entries=200, sequenceid=242, filesize=38.8 K 2024-11-19T04:42:19,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/6adfa6dd0d8e4770ab0053562e7a6c6a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/6adfa6dd0d8e4770ab0053562e7a6c6a 2024-11-19T04:42:19,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/6adfa6dd0d8e4770ab0053562e7a6c6a, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T04:42:19,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/ae4feb1754304d14be2a636a7135436c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/ae4feb1754304d14be2a636a7135436c 2024-11-19T04:42:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:19,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/ae4feb1754304d14be2a636a7135436c, entries=150, sequenceid=242, filesize=11.9 K 2024-11-19T04:42:19,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1263ms, sequenceid=242, compaction requested=true 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:19,873 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:19,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:19,875 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:19,875 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/B is initiating minor compaction (all files) 2024-11-19T04:42:19,875 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/B in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,875 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/49e2e90bdb12499d95c52df34ad01e6c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/aa482663c8c34b3ab32655915d721d87, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/54537f646d514bc289c91e831929ca33, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/6adfa6dd0d8e4770ab0053562e7a6c6a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=47.8 K 2024-11-19T04:42:19,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:19,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/A is initiating minor compaction (all files) 2024-11-19T04:42:19,875 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/A in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,875 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/96f87838819741bc85d296f24e83d405, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=138.7 K 2024-11-19T04:42:19,875 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:19,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/96f87838819741bc85d296f24e83d405, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827] 2024-11-19T04:42:19,876 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e2e90bdb12499d95c52df34ad01e6c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731991334492 2024-11-19T04:42:19,876 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f87838819741bc85d296f24e83d405, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731991334492 2024-11-19T04:42:19,876 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25cd0e819734bdf922c0613e82fe2ba, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731991335190 2024-11-19T04:42:19,876 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting aa482663c8c34b3ab32655915d721d87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731991335204 2024-11-19T04:42:19,877 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 54537f646d514bc289c91e831929ca33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731991335242 2024-11-19T04:42:19,877 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e1a78d1e63f473ca1dab049197c9d7f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731991335242 2024-11-19T04:42:19,877 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f8ffa5a24d4bac9143087186d95827, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991337443 2024-11-19T04:42:19,877 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6adfa6dd0d8e4770ab0053562e7a6c6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991337446 2024-11-19T04:42:19,886 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:19,888 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:19,889 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/584084daf0394504a712c7c7a31898ab is 50, key is test_row_0/B:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:19,891 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119660a2f3c0a904698979b70c9d30b1f74_7cef285b884a1aa14f0a62fe9ee2e32e store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:19,894 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119660a2f3c0a904698979b70c9d30b1f74_7cef285b884a1aa14f0a62fe9ee2e32e, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:19,894 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119660a2f3c0a904698979b70c9d30b1f74_7cef285b884a1aa14f0a62fe9ee2e32e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:19,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742253_1429 (size=12629) 2024-11-19T04:42:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742254_1430 (size=4469) 2024-11-19T04:42:19,900 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#A#compaction#361 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:19,900 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/ac2390fbd1db4f25b2be006d193d8564 is 175, key is test_row_0/A:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:19,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742255_1431 (size=31583) 2024-11-19T04:42:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:20,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T04:42:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:20,020 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:42:20,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:20,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:20,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:20,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:20,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:20,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:20,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111969b6d5c61ff041abb05b3e041c1dd215_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991338636/Put/seqid=0 2024-11-19T04:42:20,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742256_1432 (size=12304) 2024-11-19T04:42:20,299 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/584084daf0394504a712c7c7a31898ab as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/584084daf0394504a712c7c7a31898ab 2024-11-19T04:42:20,303 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/B of 7cef285b884a1aa14f0a62fe9ee2e32e into 584084daf0394504a712c7c7a31898ab(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:20,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:20,303 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/B, priority=12, startTime=1731991339873; duration=0sec 2024-11-19T04:42:20,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:20,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:B 2024-11-19T04:42:20,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:20,304 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:20,304 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/C is initiating minor compaction (all files) 2024-11-19T04:42:20,304 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/C in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:20,305 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/5e51c9f606374ad5a5684d66ddb7c254, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/4bfad097272748cd9aa750e67e054296, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/29d74c4f8432425fa5f2c6a471313c99, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/ae4feb1754304d14be2a636a7135436c] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=47.8 K 2024-11-19T04:42:20,305 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e51c9f606374ad5a5684d66ddb7c254, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731991334492 2024-11-19T04:42:20,305 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bfad097272748cd9aa750e67e054296, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731991335204 2024-11-19T04:42:20,305 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d74c4f8432425fa5f2c6a471313c99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731991335242 2024-11-19T04:42:20,306 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4feb1754304d14be2a636a7135436c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991337446 2024-11-19T04:42:20,307 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/ac2390fbd1db4f25b2be006d193d8564 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ac2390fbd1db4f25b2be006d193d8564 2024-11-19T04:42:20,312 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/A of 7cef285b884a1aa14f0a62fe9ee2e32e into ac2390fbd1db4f25b2be006d193d8564(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:20,312 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:20,312 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/A, priority=12, startTime=1731991339873; duration=0sec 2024-11-19T04:42:20,312 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:20,312 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:A 2024-11-19T04:42:20,314 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:20,315 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/3fbc093f4bfa4c2e87c4fa45bc06169d is 50, key is test_row_0/C:col10/1731991337446/Put/seqid=0 2024-11-19T04:42:20,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742257_1433 (size=12629) 2024-11-19T04:42:20,327 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/3fbc093f4bfa4c2e87c4fa45bc06169d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/3fbc093f4bfa4c2e87c4fa45bc06169d 2024-11-19T04:42:20,331 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/C of 7cef285b884a1aa14f0a62fe9ee2e32e into 3fbc093f4bfa4c2e87c4fa45bc06169d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:20,331 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:20,331 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/C, priority=12, startTime=1731991339873; duration=0sec 2024-11-19T04:42:20,331 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:20,331 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:C 2024-11-19T04:42:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:20,434 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111969b6d5c61ff041abb05b3e041c1dd215_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111969b6d5c61ff041abb05b3e041c1dd215_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:20,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/3fecdbacf21746c1871d7342a64f72d1, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:20,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/3fecdbacf21746c1871d7342a64f72d1 is 175, key is test_row_0/A:col10/1731991338636/Put/seqid=0 2024-11-19T04:42:20,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742258_1434 (size=31105) 2024-11-19T04:42:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:20,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:20,840 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/3fecdbacf21746c1871d7342a64f72d1 2024-11-19T04:42:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/0df146a0c8b94d708db3d7efe66e83ca is 50, key is test_row_0/B:col10/1731991338636/Put/seqid=0 2024-11-19T04:42:20,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742259_1435 (size=12151) 2024-11-19T04:42:20,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991340864, max timestamp=1731991340867Num uniques 859; 2024-11-19T04:42:20,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:20,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991340864, max timestamp=1731991340867Num uniques 859; 2024-11-19T04:42:20,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:20,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991340864, max timestamp=1731991340867Num uniques 859; 2024-11-19T04:42:20,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991400912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991400918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:20,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991400920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991400924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:21,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991401027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991401037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991401047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991401047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,252 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/0df146a0c8b94d708db3d7efe66e83ca 2024-11-19T04:42:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991401247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/daefc9b694994310a4fbacf6a287381f is 50, key is test_row_0/C:col10/1731991338636/Put/seqid=0 2024-11-19T04:42:21,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742260_1436 (size=12151) 2024-11-19T04:42:21,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991401257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991401267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991401268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991401557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991401577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991401587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991401587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:21,662 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/daefc9b694994310a4fbacf6a287381f 2024-11-19T04:42:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/3fecdbacf21746c1871d7342a64f72d1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1 2024-11-19T04:42:21,670 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1, entries=150, sequenceid=253, filesize=30.4 K 2024-11-19T04:42:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/0df146a0c8b94d708db3d7efe66e83ca as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/0df146a0c8b94d708db3d7efe66e83ca 2024-11-19T04:42:21,674 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/0df146a0c8b94d708db3d7efe66e83ca, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T04:42:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/daefc9b694994310a4fbacf6a287381f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/daefc9b694994310a4fbacf6a287381f 2024-11-19T04:42:21,678 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/daefc9b694994310a4fbacf6a287381f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T04:42:21,679 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1659ms, sequenceid=253, compaction requested=false 2024-11-19T04:42:21,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:21,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:21,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T04:42:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T04:42:21,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T04:42:21,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8900 sec 2024-11-19T04:42:21,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.8950 sec 2024-11-19T04:42:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:22,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T04:42:22,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:22,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:22,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:22,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:22,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:22,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111970b8ac832d9341e4bcd452493a88dcf2_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:22,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742261_1437 (size=14994) 2024-11-19T04:42:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991402118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991402120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991402120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991402121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991402226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991402237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991402237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991402247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991402446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991402457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991402457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991402467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:22,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111970b8ac832d9341e4bcd452493a88dcf2_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111970b8ac832d9341e4bcd452493a88dcf2_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:22,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/40bf22b1f721482c92653121ef73d5ee, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:22,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/40bf22b1f721482c92653121ef73d5ee is 175, key is test_row_0/A:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742262_1438 (size=39949) 2024-11-19T04:42:22,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991402767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991402767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991402776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991402787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:22,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:22,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/40bf22b1f721482c92653121ef73d5ee 2024-11-19T04:42:22,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/e21a3bcbcfa7443b923dd32b6336a5c7 is 50, key is test_row_0/B:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:22,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742263_1439 (size=12301) 2024-11-19T04:42:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T04:42:22,957 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:22,958 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T04:42:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T04:42:22,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:22,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:22,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T04:42:23,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T04:42:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T04:42:23,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:23,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T04:42:23,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991403291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:23,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991403291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991403297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:23,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/e21a3bcbcfa7443b923dd32b6336a5c7 2024-11-19T04:42:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991403307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:23,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/0d4824f730f2444284aec045b3a40766 is 50, key is test_row_0/C:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:23,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742264_1440 (size=12301) 2024-11-19T04:42:23,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T04:42:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:23,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36798 deadline: 1731991403537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:23,546 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 7cef285b884a1aa14f0a62fe9ee2e32e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:23,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T04:42:23,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:23,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T04:42:23,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T04:42:23,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:23,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:23,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/0d4824f730f2444284aec045b3a40766 2024-11-19T04:42:23,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/40bf22b1f721482c92653121ef73d5ee as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee 2024-11-19T04:42:23,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee, entries=200, sequenceid=293, filesize=39.0 K 2024-11-19T04:42:23,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/e21a3bcbcfa7443b923dd32b6336a5c7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/e21a3bcbcfa7443b923dd32b6336a5c7 2024-11-19T04:42:23,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/e21a3bcbcfa7443b923dd32b6336a5c7, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T04:42:23,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/0d4824f730f2444284aec045b3a40766 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/0d4824f730f2444284aec045b3a40766 2024-11-19T04:42:23,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/0d4824f730f2444284aec045b3a40766, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T04:42:23,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1659ms, sequenceid=293, compaction requested=true 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:23,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:23,745 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:23,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:23,746 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:23,746 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:23,746 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/B is initiating minor compaction (all files) 2024-11-19T04:42:23,746 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/A is initiating minor compaction (all files) 2024-11-19T04:42:23,746 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/B in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,746 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/A in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,746 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ac2390fbd1db4f25b2be006d193d8564, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=100.2 K 2024-11-19T04:42:23,746 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/584084daf0394504a712c7c7a31898ab, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/0df146a0c8b94d708db3d7efe66e83ca, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/e21a3bcbcfa7443b923dd32b6336a5c7] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=36.2 K 2024-11-19T04:42:23,746 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,746 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ac2390fbd1db4f25b2be006d193d8564, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee] 2024-11-19T04:42:23,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac2390fbd1db4f25b2be006d193d8564, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991337446 2024-11-19T04:42:23,747 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 584084daf0394504a712c7c7a31898ab, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991337446 2024-11-19T04:42:23,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fecdbacf21746c1871d7342a64f72d1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731991338630 2024-11-19T04:42:23,747 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df146a0c8b94d708db3d7efe66e83ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731991338630 2024-11-19T04:42:23,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40bf22b1f721482c92653121ef73d5ee, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731991340899 2024-11-19T04:42:23,747 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e21a3bcbcfa7443b923dd32b6336a5c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731991340910 2024-11-19T04:42:23,753 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:23,754 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:23,754 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111947050a6352c840dda5d8c398b8fd434b_7cef285b884a1aa14f0a62fe9ee2e32e store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:23,754 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/bbca7d5f46f7418ba6cafa08ecb0d7ac is 50, key is test_row_0/B:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:23,756 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111947050a6352c840dda5d8c398b8fd434b_7cef285b884a1aa14f0a62fe9ee2e32e, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:23,756 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111947050a6352c840dda5d8c398b8fd434b_7cef285b884a1aa14f0a62fe9ee2e32e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:23,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742265_1441 (size=12881) 2024-11-19T04:42:23,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742266_1442 (size=4469) 2024-11-19T04:42:23,765 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#A#compaction#369 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:23,766 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/3856dfa6071848f5923bc538bbca1394 is 175, key is test_row_0/A:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:23,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742267_1443 (size=31835) 2024-11-19T04:42:23,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:23,879 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:23,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aaeca2063833453a9f7460e098061333_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991342118/Put/seqid=0 2024-11-19T04:42:23,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742268_1444 (size=12454) 2024-11-19T04:42:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T04:42:24,169 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/bbca7d5f46f7418ba6cafa08ecb0d7ac as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/bbca7d5f46f7418ba6cafa08ecb0d7ac 2024-11-19T04:42:24,172 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/3856dfa6071848f5923bc538bbca1394 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3856dfa6071848f5923bc538bbca1394 2024-11-19T04:42:24,173 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/B of 7cef285b884a1aa14f0a62fe9ee2e32e into bbca7d5f46f7418ba6cafa08ecb0d7ac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:24,173 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:24,173 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/B, priority=13, startTime=1731991343745; duration=0sec 2024-11-19T04:42:24,173 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:24,173 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:B 2024-11-19T04:42:24,173 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:24,175 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:24,175 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/C is initiating minor compaction (all files) 2024-11-19T04:42:24,175 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/C in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:24,175 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/3fbc093f4bfa4c2e87c4fa45bc06169d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/daefc9b694994310a4fbacf6a287381f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/0d4824f730f2444284aec045b3a40766] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=36.2 K 2024-11-19T04:42:24,176 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fbc093f4bfa4c2e87c4fa45bc06169d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731991337446 2024-11-19T04:42:24,176 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting daefc9b694994310a4fbacf6a287381f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731991338630 2024-11-19T04:42:24,176 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d4824f730f2444284aec045b3a40766, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731991340910 2024-11-19T04:42:24,178 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/A of 7cef285b884a1aa14f0a62fe9ee2e32e into 3856dfa6071848f5923bc538bbca1394(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:24,178 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:24,178 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/A, priority=13, startTime=1731991343745; duration=0sec 2024-11-19T04:42:24,178 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:24,178 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:A 2024-11-19T04:42:24,183 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:24,184 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/d90478151b2b4b5895fc2caaf44e7420 is 50, key is test_row_0/C:col10/1731991342083/Put/seqid=0 2024-11-19T04:42:24,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742269_1445 (size=12881) 2024-11-19T04:42:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:24,295 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aaeca2063833453a9f7460e098061333_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aaeca2063833453a9f7460e098061333_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:24,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/172ece39b88a4f42a4423f6032ab8c43, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/172ece39b88a4f42a4423f6032ab8c43 is 175, key is test_row_0/A:col10/1731991342118/Put/seqid=0 2024-11-19T04:42:24,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742270_1446 (size=31255) 2024-11-19T04:42:24,301 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/172ece39b88a4f42a4423f6032ab8c43 2024-11-19T04:42:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/59e5be5f3566402d8f97affbca7d63a1 is 50, key is test_row_0/B:col10/1731991342118/Put/seqid=0 2024-11-19T04:42:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742271_1447 (size=12301) 2024-11-19T04:42:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:24,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. as already flushing 2024-11-19T04:42:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991404364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991344349, max timestamp=1731991344368Num uniques 859; 2024-11-19T04:42:24,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:24,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991344349, max timestamp=1731991344368Num uniques 859; 2024-11-19T04:42:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991344349, max timestamp=1731991344368Num uniques 859; 2024-11-19T04:42:24,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991404436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991404436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991404437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991404476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991404557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991404557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991404557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,593 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/d90478151b2b4b5895fc2caaf44e7420 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d90478151b2b4b5895fc2caaf44e7420 2024-11-19T04:42:24,598 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/C of 7cef285b884a1aa14f0a62fe9ee2e32e into d90478151b2b4b5895fc2caaf44e7420(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:24,598 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:24,598 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/C, priority=13, startTime=1731991343745; duration=0sec 2024-11-19T04:42:24,598 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:24,598 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:C 2024-11-19T04:42:24,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991404687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,712 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/59e5be5f3566402d8f97affbca7d63a1 2024-11-19T04:42:24,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/f922f18961e14568aede5352c1fab88b is 50, key is test_row_0/C:col10/1731991342118/Put/seqid=0 2024-11-19T04:42:24,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742272_1448 (size=12301) 2024-11-19T04:42:24,752 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/f922f18961e14568aede5352c1fab88b 2024-11-19T04:42:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/172ece39b88a4f42a4423f6032ab8c43 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43 2024-11-19T04:42:24,762 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43, entries=150, sequenceid=304, filesize=30.5 K 2024-11-19T04:42:24,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/59e5be5f3566402d8f97affbca7d63a1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/59e5be5f3566402d8f97affbca7d63a1 2024-11-19T04:42:24,769 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/59e5be5f3566402d8f97affbca7d63a1, entries=150, sequenceid=304, filesize=12.0 K 2024-11-19T04:42:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/f922f18961e14568aede5352c1fab88b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f922f18961e14568aede5352c1fab88b 2024-11-19T04:42:24,774 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f922f18961e14568aede5352c1fab88b, entries=150, sequenceid=304, filesize=12.0 K 2024-11-19T04:42:24,775 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7cef285b884a1aa14f0a62fe9ee2e32e in 896ms, sequenceid=304, compaction requested=false 2024-11-19T04:42:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T04:42:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T04:42:24,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T04:42:24,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8170 sec 2024-11-19T04:42:24,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8230 sec 2024-11-19T04:42:24,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-19T04:42:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:24,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:24,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:24,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:24,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:24,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:24,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:24,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fbfcc511cd244d98b44c43bc913c98f_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:24,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991404820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991404832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991404838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742273_1449 (size=17534) 2024-11-19T04:42:24,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:24,863 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192fbfcc511cd244d98b44c43bc913c98f_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fbfcc511cd244d98b44c43bc913c98f_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:24,864 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:24,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d is 175, key is test_row_0/A:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742274_1450 (size=48639) 2024-11-19T04:42:24,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d 2024-11-19T04:42:24,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/9ab44003868a435fa9380b0e0dc69e32 is 50, key is test_row_0/B:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:24,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742275_1451 (size=12301) 2024-11-19T04:42:24,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/9ab44003868a435fa9380b0e0dc69e32 2024-11-19T04:42:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36838 deadline: 1731991404947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36850 deadline: 1731991404957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36758 deadline: 1731991404957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:24,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/11f64d8191bc44f896c973c97cec7be7 is 50, key is test_row_0/C:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:25,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742276_1452 (size=12301) 2024-11-19T04:42:25,013 INFO [Thread-1693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1693. 2024-11-19T04:42:25,013 DEBUG [Thread-1693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,013 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,013 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/11f64d8191bc44f896c973c97cec7be7 2024-11-19T04:42:25,016 INFO [Thread-1689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1689. 2024-11-19T04:42:25,016 DEBUG [Thread-1689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,016 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,016 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d 2024-11-19T04:42:25,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731991405007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:42:25,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d, entries=250, sequenceid=343, filesize=47.5 K 2024-11-19T04:42:25,025 INFO [Thread-1685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1685. 2024-11-19T04:42:25,025 DEBUG [Thread-1685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,025 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/9ab44003868a435fa9380b0e0dc69e32 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/9ab44003868a435fa9380b0e0dc69e32 2024-11-19T04:42:25,025 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/9ab44003868a435fa9380b0e0dc69e32, entries=150, sequenceid=343, filesize=12.0 K 2024-11-19T04:42:25,032 INFO [Thread-1687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1687. 2024-11-19T04:42:25,032 DEBUG [Thread-1687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,032 DEBUG [Thread-1687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,032 DEBUG [Thread-1687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,032 INFO [Thread-1691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1691. 2024-11-19T04:42:25,033 DEBUG [Thread-1691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,033 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,033 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/11f64d8191bc44f896c973c97cec7be7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/11f64d8191bc44f896c973c97cec7be7 2024-11-19T04:42:25,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/11f64d8191bc44f896c973c97cec7be7, entries=150, sequenceid=343, filesize=12.0 K 2024-11-19T04:42:25,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 7cef285b884a1aa14f0a62fe9ee2e32e in 247ms, sequenceid=343, compaction requested=true 2024-11-19T04:42:25,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:25,039 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:25,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:25,040 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cef285b884a1aa14f0a62fe9ee2e32e:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:25,042 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:25,042 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:25,042 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/A is initiating minor compaction (all files) 2024-11-19T04:42:25,042 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/B is initiating minor compaction (all files) 2024-11-19T04:42:25,042 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/A in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:25,042 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/B in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:25,042 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3856dfa6071848f5923bc538bbca1394, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=109.1 K 2024-11-19T04:42:25,042 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/bbca7d5f46f7418ba6cafa08ecb0d7ac, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/59e5be5f3566402d8f97affbca7d63a1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/9ab44003868a435fa9380b0e0dc69e32] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=36.6 K 2024-11-19T04:42:25,042 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:25,042 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3856dfa6071848f5923bc538bbca1394, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d] 2024-11-19T04:42:25,042 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting bbca7d5f46f7418ba6cafa08ecb0d7ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731991340910 2024-11-19T04:42:25,043 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3856dfa6071848f5923bc538bbca1394, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731991340910 2024-11-19T04:42:25,043 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e5be5f3566402d8f97affbca7d63a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731991342117 2024-11-19T04:42:25,045 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172ece39b88a4f42a4423f6032ab8c43, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731991342117 2024-11-19T04:42:25,045 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab44003868a435fa9380b0e0dc69e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731991344413 2024-11-19T04:42:25,045 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca5f59e1b1d4e58b0cfdda6ddb9d35d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731991344393 2024-11-19T04:42:25,065 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:25,067 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:25,068 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/36d6fda08aa44f4289800239f7d01e19 is 50, key is test_row_0/B:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:25,069 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e9eadf4e431c416db72cdb012640d1e1_7cef285b884a1aa14f0a62fe9ee2e32e store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:25,072 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e9eadf4e431c416db72cdb012640d1e1_7cef285b884a1aa14f0a62fe9ee2e32e, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:25,072 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e9eadf4e431c416db72cdb012640d1e1_7cef285b884a1aa14f0a62fe9ee2e32e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T04:42:25,107 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742277_1453 (size=12983) 2024-11-19T04:42:25,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742278_1454 (size=4469) 2024-11-19T04:42:25,117 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/36d6fda08aa44f4289800239f7d01e19 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/36d6fda08aa44f4289800239f7d01e19 2024-11-19T04:42:25,125 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/B of 7cef285b884a1aa14f0a62fe9ee2e32e into 36d6fda08aa44f4289800239f7d01e19(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:25,125 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:25,125 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/B, priority=13, startTime=1731991345040; duration=0sec 2024-11-19T04:42:25,125 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:25,125 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:B 2024-11-19T04:42:25,125 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:25,126 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:25,126 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 7cef285b884a1aa14f0a62fe9ee2e32e/C is initiating minor compaction (all files) 2024-11-19T04:42:25,126 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cef285b884a1aa14f0a62fe9ee2e32e/C in TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:25,126 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d90478151b2b4b5895fc2caaf44e7420, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f922f18961e14568aede5352c1fab88b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/11f64d8191bc44f896c973c97cec7be7] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp, totalSize=36.6 K 2024-11-19T04:42:25,127 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d90478151b2b4b5895fc2caaf44e7420, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731991340910 2024-11-19T04:42:25,127 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting f922f18961e14568aede5352c1fab88b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731991342117 2024-11-19T04:42:25,127 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f64d8191bc44f896c973c97cec7be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731991344413 2024-11-19T04:42:25,139 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:25,139 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e1838afa166f4e06b3188467f9dfe180 is 50, key is test_row_0/C:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:25,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742279_1455 (size=12983) 2024-11-19T04:42:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:25,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:25,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:25,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:25,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:25,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:25,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:25,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:25,170 INFO [Thread-1676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1676. 2024-11-19T04:42:25,171 DEBUG [Thread-1676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,171 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,171 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a928158548ac40ca97e5f0c395b09bac_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991345168/Put/seqid=0 2024-11-19T04:42:25,178 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-11-19T04:42:25,178 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,178 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,179 INFO [Thread-1678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1678. 2024-11-19T04:42:25,179 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,179 DEBUG [Thread-1678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,179 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,179 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742280_1456 (size=12454) 2024-11-19T04:42:25,515 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cef285b884a1aa14f0a62fe9ee2e32e#A#compaction#379 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:25,515 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/f18db27129ce46b09bd0e787c2b5d442 is 175, key is test_row_0/A:col10/1731991344789/Put/seqid=0 2024-11-19T04:42:25,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742281_1457 (size=31937) 2024-11-19T04:42:25,539 INFO [Thread-1674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1674. 2024-11-19T04:42:25,539 DEBUG [Thread-1674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:25,539 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,539 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:25,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:25,552 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e1838afa166f4e06b3188467f9dfe180 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e1838afa166f4e06b3188467f9dfe180 2024-11-19T04:42:25,556 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/C of 7cef285b884a1aa14f0a62fe9ee2e32e into e1838afa166f4e06b3188467f9dfe180(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:25,556 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:25,556 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/C, priority=13, startTime=1731991345040; duration=0sec 2024-11-19T04:42:25,556 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:25,556 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:C 2024-11-19T04:42:25,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:25,586 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a928158548ac40ca97e5f0c395b09bac_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a928158548ac40ca97e5f0c395b09bac_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:25,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/2218086371374bb5a1f22fab6a30d4ab, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:25,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/2218086371374bb5a1f22fab6a30d4ab is 175, key is test_row_0/A:col10/1731991345168/Put/seqid=0 2024-11-19T04:42:25,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742282_1458 (size=31255) 2024-11-19T04:42:25,923 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/f18db27129ce46b09bd0e787c2b5d442 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f18db27129ce46b09bd0e787c2b5d442 2024-11-19T04:42:25,927 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cef285b884a1aa14f0a62fe9ee2e32e/A of 7cef285b884a1aa14f0a62fe9ee2e32e into f18db27129ce46b09bd0e787c2b5d442(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:25,927 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:25,927 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., storeName=7cef285b884a1aa14f0a62fe9ee2e32e/A, priority=13, startTime=1731991345039; duration=0sec 2024-11-19T04:42:25,927 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:25,927 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cef285b884a1aa14f0a62fe9ee2e32e:A 2024-11-19T04:42:25,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/2218086371374bb5a1f22fab6a30d4ab 2024-11-19T04:42:25,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/f20b4be53f664356a08784130cadfb4c is 50, key is test_row_0/B:col10/1731991345168/Put/seqid=0 2024-11-19T04:42:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742283_1459 (size=12301) 2024-11-19T04:42:26,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/f20b4be53f664356a08784130cadfb4c 2024-11-19T04:42:26,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/81b5cca65d2e4d54946105407de244f8 is 50, key is test_row_0/C:col10/1731991345168/Put/seqid=0 2024-11-19T04:42:26,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742284_1460 (size=12301) 2024-11-19T04:42:26,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/81b5cca65d2e4d54946105407de244f8 2024-11-19T04:42:26,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/2218086371374bb5a1f22fab6a30d4ab as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/2218086371374bb5a1f22fab6a30d4ab 2024-11-19T04:42:26,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/2218086371374bb5a1f22fab6a30d4ab, entries=150, sequenceid=354, filesize=30.5 K 2024-11-19T04:42:26,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/f20b4be53f664356a08784130cadfb4c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/f20b4be53f664356a08784130cadfb4c 2024-11-19T04:42:26,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/f20b4be53f664356a08784130cadfb4c, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T04:42:26,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/81b5cca65d2e4d54946105407de244f8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/81b5cca65d2e4d54946105407de244f8 2024-11-19T04:42:26,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/81b5cca65d2e4d54946105407de244f8, entries=150, sequenceid=354, filesize=12.0 K 2024-11-19T04:42:26,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1652ms, sequenceid=354, compaction requested=false 2024-11-19T04:42:26,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cef285b884a1aa14f0a62fe9ee2e32e: 2024-11-19T04:42:31,219 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T04:42:33,598 INFO [Thread-1682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1682. 2024-11-19T04:42:33,598 DEBUG [Thread-1682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:33,598 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:33,598 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:33,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1029 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3085 rows 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1026 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3076 rows 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1027 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3080 rows 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1026 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3074 rows 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1025 2024-11-19T04:42:33,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3072 rows 2024-11-19T04:42:33,599 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:42:33,599 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:42:33,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:33,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:33,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:33,600 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T04:42:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T04:42:33,602 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991353602"}]},"ts":"1731991353602"} 2024-11-19T04:42:33,603 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T04:42:33,603 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T04:42:33,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:42:33,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, UNASSIGN}] 2024-11-19T04:42:33,606 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, UNASSIGN 2024-11-19T04:42:33,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:33,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, UNASSIGN because future has completed 2024-11-19T04:42:33,607 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:42:33,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:42:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T04:42:33,758 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:33,758 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 7cef285b884a1aa14f0a62fe9ee2e32e, disabling compactions & flushes 2024-11-19T04:42:33,759 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. after waiting 0 ms 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:33,759 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 7cef285b884a1aa14f0a62fe9ee2e32e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=A 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=B 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7cef285b884a1aa14f0a62fe9ee2e32e, store=C 2024-11-19T04:42:33,759 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:33,764 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a375efca0ee74c368f67e579a28b9843_7cef285b884a1aa14f0a62fe9ee2e32e is 50, key is test_row_0/A:col10/1731991345177/Put/seqid=0 2024-11-19T04:42:33,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742285_1461 (size=9914) 2024-11-19T04:42:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T04:42:34,167 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:34,171 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a375efca0ee74c368f67e579a28b9843_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a375efca0ee74c368f67e579a28b9843_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:34,172 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/79b74f637c364ec692345e394591c48a, store: [table=TestAcidGuarantees family=A region=7cef285b884a1aa14f0a62fe9ee2e32e] 2024-11-19T04:42:34,172 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/79b74f637c364ec692345e394591c48a is 175, key is test_row_0/A:col10/1731991345177/Put/seqid=0 2024-11-19T04:42:34,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742286_1462 (size=22561) 2024-11-19T04:42:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T04:42:34,576 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/79b74f637c364ec692345e394591c48a 2024-11-19T04:42:34,581 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/20eca4580f4c4c3bb1d7eef2b87ab00a is 50, key is test_row_0/B:col10/1731991345177/Put/seqid=0 2024-11-19T04:42:34,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742287_1463 (size=9857) 2024-11-19T04:42:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T04:42:34,985 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/20eca4580f4c4c3bb1d7eef2b87ab00a 2024-11-19T04:42:34,991 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e6fd291088ed434fb12ad59ec0dde01f is 50, key is test_row_0/C:col10/1731991345177/Put/seqid=0 2024-11-19T04:42:34,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742288_1464 (size=9857) 2024-11-19T04:42:35,191 WARN [regionserver/08a7f35e60d4:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-19T04:42:35,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/5e447185343a4ca9a17a2474b7667a0d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ad916a40f1ea4ceca37964cff6dfb503, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/96f87838819741bc85d296f24e83d405, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ac2390fbd1db4f25b2be006d193d8564, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3856dfa6071848f5923bc538bbca1394, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d] to archive 2024-11-19T04:42:35,193 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:42:35,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/44272e64efb241949a36f683766e402f 2024-11-19T04:42:35,195 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/bbcb9a06d8b34061b8532899a97553cc 2024-11-19T04:42:35,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/db5630e5246a4991876b7a118214c120 2024-11-19T04:42:35,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/5e447185343a4ca9a17a2474b7667a0d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/5e447185343a4ca9a17a2474b7667a0d 2024-11-19T04:42:35,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f368fc9eaca94e06802eb4d9a7880476 2024-11-19T04:42:35,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/acf85a9a1c2d456abff818a5ac389704 2024-11-19T04:42:35,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d7e088cd38d64a72b77fe7832941050c 2024-11-19T04:42:35,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ad916a40f1ea4ceca37964cff6dfb503 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ad916a40f1ea4ceca37964cff6dfb503 2024-11-19T04:42:35,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/37e4862bdefe4bac8e41207b8a654d80 2024-11-19T04:42:35,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/d66159a3670e46618b4f19c095709542 2024-11-19T04:42:35,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/96f87838819741bc85d296f24e83d405 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/96f87838819741bc85d296f24e83d405 2024-11-19T04:42:35,203 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/c25cd0e819734bdf922c0613e82fe2ba 2024-11-19T04:42:35,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/9e1a78d1e63f473ca1dab049197c9d7f 2024-11-19T04:42:35,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/80f8ffa5a24d4bac9143087186d95827 2024-11-19T04:42:35,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ac2390fbd1db4f25b2be006d193d8564 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/ac2390fbd1db4f25b2be006d193d8564 2024-11-19T04:42:35,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3fecdbacf21746c1871d7342a64f72d1 2024-11-19T04:42:35,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/40bf22b1f721482c92653121ef73d5ee 2024-11-19T04:42:35,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3856dfa6071848f5923bc538bbca1394 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/3856dfa6071848f5923bc538bbca1394 2024-11-19T04:42:35,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/172ece39b88a4f42a4423f6032ab8c43 2024-11-19T04:42:35,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/eca5f59e1b1d4e58b0cfdda6ddb9d35d 2024-11-19T04:42:35,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/3e15a865afcd472da0d878d1d727e530, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/8a0f8825f6774001a9d2170fe81aac38, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/23f59f025d554806823c16bb842700b7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d18bed6cb4234ac7a9746941c0c4a14b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/7b06690b1d2f47cbb6b42075ff42a786, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/b51ac2d42cca41eebeb35502e8736dc6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d7532186ab7642e5bdfdc878ceae9ea0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/c6096c46c7fe493190a925dae4cab785, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d869965ee4ca42db8a8f07e8945b5c59, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/49e2e90bdb12499d95c52df34ad01e6c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/348540674d0d41beae67f1922a208db6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/aa482663c8c34b3ab32655915d721d87, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/54537f646d514bc289c91e831929ca33, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/584084daf0394504a712c7c7a31898ab, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/6adfa6dd0d8e4770ab0053562e7a6c6a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/0df146a0c8b94d708db3d7efe66e83ca, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/bbca7d5f46f7418ba6cafa08ecb0d7ac, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/e21a3bcbcfa7443b923dd32b6336a5c7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/59e5be5f3566402d8f97affbca7d63a1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/9ab44003868a435fa9380b0e0dc69e32] to archive 2024-11-19T04:42:35,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:42:35,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/3e15a865afcd472da0d878d1d727e530 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/3e15a865afcd472da0d878d1d727e530 2024-11-19T04:42:35,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/8a0f8825f6774001a9d2170fe81aac38 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/8a0f8825f6774001a9d2170fe81aac38 2024-11-19T04:42:35,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/23f59f025d554806823c16bb842700b7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/23f59f025d554806823c16bb842700b7 2024-11-19T04:42:35,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d18bed6cb4234ac7a9746941c0c4a14b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d18bed6cb4234ac7a9746941c0c4a14b 2024-11-19T04:42:35,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/7b06690b1d2f47cbb6b42075ff42a786 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/7b06690b1d2f47cbb6b42075ff42a786 2024-11-19T04:42:35,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/b51ac2d42cca41eebeb35502e8736dc6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/b51ac2d42cca41eebeb35502e8736dc6 2024-11-19T04:42:35,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d7532186ab7642e5bdfdc878ceae9ea0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d7532186ab7642e5bdfdc878ceae9ea0 2024-11-19T04:42:35,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/c6096c46c7fe493190a925dae4cab785 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/c6096c46c7fe493190a925dae4cab785 2024-11-19T04:42:35,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d869965ee4ca42db8a8f07e8945b5c59 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/d869965ee4ca42db8a8f07e8945b5c59 2024-11-19T04:42:35,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/49e2e90bdb12499d95c52df34ad01e6c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/49e2e90bdb12499d95c52df34ad01e6c 2024-11-19T04:42:35,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/348540674d0d41beae67f1922a208db6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/348540674d0d41beae67f1922a208db6 2024-11-19T04:42:35,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/aa482663c8c34b3ab32655915d721d87 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/aa482663c8c34b3ab32655915d721d87 2024-11-19T04:42:35,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/54537f646d514bc289c91e831929ca33 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/54537f646d514bc289c91e831929ca33 2024-11-19T04:42:35,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/584084daf0394504a712c7c7a31898ab to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/584084daf0394504a712c7c7a31898ab 2024-11-19T04:42:35,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/6adfa6dd0d8e4770ab0053562e7a6c6a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/6adfa6dd0d8e4770ab0053562e7a6c6a 2024-11-19T04:42:35,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/0df146a0c8b94d708db3d7efe66e83ca to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/0df146a0c8b94d708db3d7efe66e83ca 2024-11-19T04:42:35,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/bbca7d5f46f7418ba6cafa08ecb0d7ac to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/bbca7d5f46f7418ba6cafa08ecb0d7ac 2024-11-19T04:42:35,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/e21a3bcbcfa7443b923dd32b6336a5c7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/e21a3bcbcfa7443b923dd32b6336a5c7 2024-11-19T04:42:35,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/59e5be5f3566402d8f97affbca7d63a1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/59e5be5f3566402d8f97affbca7d63a1 2024-11-19T04:42:35,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/9ab44003868a435fa9380b0e0dc69e32 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/9ab44003868a435fa9380b0e0dc69e32 2024-11-19T04:42:35,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/6d5e43f5e97e4b199d2fa5954d91d32c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e4f22d73479d4185bc9a35b24803c832, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/bdf9c74df8ca4c2d8f917a78c72eb166, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/17bd0c97caaf438888b9940e0b812e74, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d4b8fdc086d24a69b66d045b6978bab6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/8e53ac18009d41968d32168d74efa953, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f0693d813907463596c15824aeb0ae26, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/1252b4d0613c40f3909495df3fb68d27, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/b26bfd695bfc4792ac5b328d182c4aad, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/5e51c9f606374ad5a5684d66ddb7c254, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/c8923e5e00934658981ef1da954a6141, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/4bfad097272748cd9aa750e67e054296, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/29d74c4f8432425fa5f2c6a471313c99, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/3fbc093f4bfa4c2e87c4fa45bc06169d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/ae4feb1754304d14be2a636a7135436c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/daefc9b694994310a4fbacf6a287381f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d90478151b2b4b5895fc2caaf44e7420, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/0d4824f730f2444284aec045b3a40766, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f922f18961e14568aede5352c1fab88b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/11f64d8191bc44f896c973c97cec7be7] to archive 2024-11-19T04:42:35,231 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:42:35,232 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/6d5e43f5e97e4b199d2fa5954d91d32c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/6d5e43f5e97e4b199d2fa5954d91d32c 2024-11-19T04:42:35,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e4f22d73479d4185bc9a35b24803c832 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e4f22d73479d4185bc9a35b24803c832 2024-11-19T04:42:35,234 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/bdf9c74df8ca4c2d8f917a78c72eb166 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/bdf9c74df8ca4c2d8f917a78c72eb166 2024-11-19T04:42:35,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/17bd0c97caaf438888b9940e0b812e74 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/17bd0c97caaf438888b9940e0b812e74 2024-11-19T04:42:35,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d4b8fdc086d24a69b66d045b6978bab6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d4b8fdc086d24a69b66d045b6978bab6 2024-11-19T04:42:35,236 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/8e53ac18009d41968d32168d74efa953 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/8e53ac18009d41968d32168d74efa953 2024-11-19T04:42:35,237 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f0693d813907463596c15824aeb0ae26 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f0693d813907463596c15824aeb0ae26 2024-11-19T04:42:35,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/1252b4d0613c40f3909495df3fb68d27 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/1252b4d0613c40f3909495df3fb68d27 2024-11-19T04:42:35,238 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/b26bfd695bfc4792ac5b328d182c4aad to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/b26bfd695bfc4792ac5b328d182c4aad 2024-11-19T04:42:35,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/5e51c9f606374ad5a5684d66ddb7c254 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/5e51c9f606374ad5a5684d66ddb7c254 2024-11-19T04:42:35,240 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/c8923e5e00934658981ef1da954a6141 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/c8923e5e00934658981ef1da954a6141 2024-11-19T04:42:35,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/4bfad097272748cd9aa750e67e054296 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/4bfad097272748cd9aa750e67e054296 2024-11-19T04:42:35,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/29d74c4f8432425fa5f2c6a471313c99 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/29d74c4f8432425fa5f2c6a471313c99 2024-11-19T04:42:35,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/3fbc093f4bfa4c2e87c4fa45bc06169d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/3fbc093f4bfa4c2e87c4fa45bc06169d 2024-11-19T04:42:35,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/ae4feb1754304d14be2a636a7135436c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/ae4feb1754304d14be2a636a7135436c 2024-11-19T04:42:35,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/daefc9b694994310a4fbacf6a287381f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/daefc9b694994310a4fbacf6a287381f 2024-11-19T04:42:35,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d90478151b2b4b5895fc2caaf44e7420 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/d90478151b2b4b5895fc2caaf44e7420 2024-11-19T04:42:35,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/0d4824f730f2444284aec045b3a40766 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/0d4824f730f2444284aec045b3a40766 2024-11-19T04:42:35,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f922f18961e14568aede5352c1fab88b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/f922f18961e14568aede5352c1fab88b 2024-11-19T04:42:35,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/08a7f35e60d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/11f64d8191bc44f896c973c97cec7be7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/11f64d8191bc44f896c973c97cec7be7 2024-11-19T04:42:35,395 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e6fd291088ed434fb12ad59ec0dde01f 2024-11-19T04:42:35,399 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/A/79b74f637c364ec692345e394591c48a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/79b74f637c364ec692345e394591c48a 2024-11-19T04:42:35,402 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/79b74f637c364ec692345e394591c48a, entries=100, sequenceid=363, filesize=22.0 K 2024-11-19T04:42:35,403 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/B/20eca4580f4c4c3bb1d7eef2b87ab00a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/20eca4580f4c4c3bb1d7eef2b87ab00a 2024-11-19T04:42:35,406 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/20eca4580f4c4c3bb1d7eef2b87ab00a, entries=100, sequenceid=363, filesize=9.6 K 2024-11-19T04:42:35,406 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/.tmp/C/e6fd291088ed434fb12ad59ec0dde01f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e6fd291088ed434fb12ad59ec0dde01f 2024-11-19T04:42:35,410 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e6fd291088ed434fb12ad59ec0dde01f, entries=100, sequenceid=363, filesize=9.6 K 2024-11-19T04:42:35,410 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1651ms, sequenceid=363, compaction requested=true 2024-11-19T04:42:35,418 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-19T04:42:35,418 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. 2024-11-19T04:42:35,418 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 7cef285b884a1aa14f0a62fe9ee2e32e: Waiting for close lock at 1731991353758Running coprocessor pre-close hooks at 1731991353758Disabling compacts and flushes for region at 1731991353758Disabling writes for close at 1731991353759 (+1 ms)Obtaining lock to block concurrent updates at 1731991353759Preparing flush snapshotting stores in 7cef285b884a1aa14f0a62fe9ee2e32e at 1731991353759Finished memstore snapshotting TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731991353759Flushing stores of TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e. at 1731991353760 (+1 ms)Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/A: creating writer at 1731991353760Flushing mob file 7cef285b884a1aa14f0a62fe9ee2e32e/A: appending metadata at 1731991353764 (+4 ms)Flushing mob file 7cef285b884a1aa14f0a62fe9ee2e32e/A: closing flushed file at 1731991353764Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/A: appending metadata at 1731991354172 (+408 ms)Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/A: closing flushed file at 1731991354172Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/B: creating writer at 1731991354579 (+407 ms)Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/B: appending metadata at 1731991354581 (+2 ms)Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/B: closing flushed file at 1731991354581Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/C: creating writer at 1731991354988 (+407 ms)Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/C: appending metadata at 1731991354991 (+3 ms)Flushing 7cef285b884a1aa14f0a62fe9ee2e32e/C: closing flushed file at 1731991354991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5516cade: reopening flushed file at 1731991355398 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1095d687: reopening flushed file at 1731991355402 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37270ff8: reopening flushed file at 1731991355406 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7cef285b884a1aa14f0a62fe9ee2e32e in 1651ms, sequenceid=363, compaction requested=true at 1731991355410 (+4 ms)Writing region close event to WAL at 1731991355414 (+4 ms)Running coprocessor post-close hooks at 1731991355418 (+4 ms)Closed at 1731991355418 2024-11-19T04:42:35,420 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,421 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=7cef285b884a1aa14f0a62fe9ee2e32e, regionState=CLOSED 2024-11-19T04:42:35,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:42:35,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T04:42:35,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 7cef285b884a1aa14f0a62fe9ee2e32e, server=08a7f35e60d4,32975,1731991233019 in 1.8160 sec 2024-11-19T04:42:35,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-19T04:42:35,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cef285b884a1aa14f0a62fe9ee2e32e, UNASSIGN in 1.8190 sec 2024-11-19T04:42:35,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T04:42:35,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8220 sec 2024-11-19T04:42:35,428 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991355428"}]},"ts":"1731991355428"} 2024-11-19T04:42:35,430 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T04:42:35,430 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T04:42:35,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8310 sec 2024-11-19T04:42:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T04:42:35,747 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:35,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T04:42:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T04:42:35,750 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,752 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,753 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/recovered.edits] 2024-11-19T04:42:35,755 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/2218086371374bb5a1f22fab6a30d4ab to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/2218086371374bb5a1f22fab6a30d4ab 2024-11-19T04:42:35,756 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/79b74f637c364ec692345e394591c48a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/79b74f637c364ec692345e394591c48a 2024-11-19T04:42:35,757 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f18db27129ce46b09bd0e787c2b5d442 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/A/f18db27129ce46b09bd0e787c2b5d442 2024-11-19T04:42:35,758 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/20eca4580f4c4c3bb1d7eef2b87ab00a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/20eca4580f4c4c3bb1d7eef2b87ab00a 2024-11-19T04:42:35,759 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/36d6fda08aa44f4289800239f7d01e19 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/36d6fda08aa44f4289800239f7d01e19 2024-11-19T04:42:35,760 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/f20b4be53f664356a08784130cadfb4c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/B/f20b4be53f664356a08784130cadfb4c 2024-11-19T04:42:35,762 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/81b5cca65d2e4d54946105407de244f8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/81b5cca65d2e4d54946105407de244f8 2024-11-19T04:42:35,763 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e1838afa166f4e06b3188467f9dfe180 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e1838afa166f4e06b3188467f9dfe180 2024-11-19T04:42:35,764 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e6fd291088ed434fb12ad59ec0dde01f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/C/e6fd291088ed434fb12ad59ec0dde01f 2024-11-19T04:42:35,766 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/recovered.edits/366.seqid to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e/recovered.edits/366.seqid 2024-11-19T04:42:35,766 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,766 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T04:42:35,767 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T04:42:35,767 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T04:42:35,769 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191d43ee99d01748d5bc5de22ad3ecf7f5_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191d43ee99d01748d5bc5de22ad3ecf7f5_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,770 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fbfcc511cd244d98b44c43bc913c98f_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192fbfcc511cd244d98b44c43bc913c98f_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,770 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195dc62dc28d3e4205b220f359755e1f31_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195dc62dc28d3e4205b220f359755e1f31_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,771 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119624a95c0b195498e8529670536e90db1_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119624a95c0b195498e8529670536e90db1_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,772 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119686678dea34944c38250083b90594cd6_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119686678dea34944c38250083b90594cd6_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,773 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111969b6d5c61ff041abb05b3e041c1dd215_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111969b6d5c61ff041abb05b3e041c1dd215_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,773 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111970b8ac832d9341e4bcd452493a88dcf2_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111970b8ac832d9341e4bcd452493a88dcf2_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,774 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111971ede7ba25714fac9c034714ce5b1151_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111971ede7ba25714fac9c034714ce5b1151_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,775 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197d4a92be05aa40e6949d99c794a9ca55_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197d4a92be05aa40e6949d99c794a9ca55_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a375efca0ee74c368f67e579a28b9843_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a375efca0ee74c368f67e579a28b9843_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,776 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a928158548ac40ca97e5f0c395b09bac_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a928158548ac40ca97e5f0c395b09bac_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,777 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a971df0ae95a41bb88860ad232d67e8e_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a971df0ae95a41bb88860ad232d67e8e_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,778 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aaeca2063833453a9f7460e098061333_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119aaeca2063833453a9f7460e098061333_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,779 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ae1a5466585344f594c910d66c81728a_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ae1a5466585344f594c910d66c81728a_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,780 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119caed8f3ad05043a2b386d54014b3f4ad_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119caed8f3ad05043a2b386d54014b3f4ad_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,781 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e364587e41eb4da29f22b181f7dcdd7b_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119e364587e41eb4da29f22b181f7dcdd7b_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,782 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eb22b998451c47b397860f369a229fee_7cef285b884a1aa14f0a62fe9ee2e32e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119eb22b998451c47b397860f369a229fee_7cef285b884a1aa14f0a62fe9ee2e32e 2024-11-19T04:42:35,782 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T04:42:35,784 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,786 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T04:42:35,787 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T04:42:35,788 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,788 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T04:42:35,788 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731991355788"}]},"ts":"9223372036854775807"} 2024-11-19T04:42:35,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T04:42:35,790 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7cef285b884a1aa14f0a62fe9ee2e32e, NAME => 'TestAcidGuarantees,,1731991321896.7cef285b884a1aa14f0a62fe9ee2e32e.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T04:42:35,790 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T04:42:35,790 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731991355790"}]},"ts":"9223372036854775807"} 2024-11-19T04:42:35,791 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T04:42:35,792 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-19T04:42:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T04:42:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T04:42:35,857 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:35,866 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=435 (was 512), ProcessCount=11 (was 11), AvailableMemoryMB=11521 (was 12108) 2024-11-19T04:42:35,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=435, ProcessCount=11, AvailableMemoryMB=11520 2024-11-19T04:42:35,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:42:35,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:42:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T04:42:35,878 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T04:42:35,878 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:35,878 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-19T04:42:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T04:42:35,878 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T04:42:35,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742289_1465 (size=1061) 2024-11-19T04:42:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T04:42:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T04:42:36,285 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:42:36,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742290_1466 (size=53) 2024-11-19T04:42:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T04:42:36,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:42:36,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 728b08bae7a306a40f52d977dcb73a88, disabling compactions & flushes 2024-11-19T04:42:36,690 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:36,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:36,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. after waiting 0 ms 2024-11-19T04:42:36,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:36,690 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:36,690 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 728b08bae7a306a40f52d977dcb73a88: Waiting for close lock at 1731991356690Disabling compacts and flushes for region at 1731991356690Disabling writes for close at 1731991356690Writing region close event to WAL at 1731991356690Closed at 1731991356690 2024-11-19T04:42:36,691 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T04:42:36,691 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731991356691"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731991356691"}]},"ts":"1731991356691"} 2024-11-19T04:42:36,692 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T04:42:36,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T04:42:36,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991356693"}]},"ts":"1731991356693"} 2024-11-19T04:42:36,694 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T04:42:36,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, ASSIGN}] 2024-11-19T04:42:36,695 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, ASSIGN 2024-11-19T04:42:36,696 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, ASSIGN; state=OFFLINE, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=false 2024-11-19T04:42:36,847 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=728b08bae7a306a40f52d977dcb73a88, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:36,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, ASSIGN because future has completed 2024-11-19T04:42:36,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:42:37,003 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:37,003 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:42:37,004 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,004 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:42:37,004 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,004 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,005 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,006 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:37,006 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 728b08bae7a306a40f52d977dcb73a88 columnFamilyName A 2024-11-19T04:42:37,006 DEBUG [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:37,006 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.HStore(327): Store=728b08bae7a306a40f52d977dcb73a88/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:37,006 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,007 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:37,007 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 728b08bae7a306a40f52d977dcb73a88 columnFamilyName B 2024-11-19T04:42:37,007 DEBUG [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:37,008 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.HStore(327): Store=728b08bae7a306a40f52d977dcb73a88/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:37,008 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,009 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:42:37,009 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 728b08bae7a306a40f52d977dcb73a88 columnFamilyName C 2024-11-19T04:42:37,009 DEBUG [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:42:37,009 INFO [StoreOpener-728b08bae7a306a40f52d977dcb73a88-1 {}] regionserver.HStore(327): Store=728b08bae7a306a40f52d977dcb73a88/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:42:37,009 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:37,009 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,010 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,010 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,011 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,011 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,011 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:42:37,012 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,013 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:42:37,014 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 728b08bae7a306a40f52d977dcb73a88; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61438508, jitterRate=-0.08449488878250122}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:42:37,014 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:37,014 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 728b08bae7a306a40f52d977dcb73a88: Running coprocessor pre-open hook at 1731991357004Writing region info on filesystem at 1731991357004Initializing all the Stores at 1731991357004Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991357004Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991357005 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991357005Cleaning up temporary data from old regions at 1731991357011 (+6 ms)Running coprocessor post-open hooks at 1731991357014 (+3 ms)Region opened successfully at 1731991357014 2024-11-19T04:42:37,015 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., pid=126, masterSystemTime=1731991357000 2024-11-19T04:42:37,016 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:37,016 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:37,017 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=728b08bae7a306a40f52d977dcb73a88, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T04:42:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:42:37,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T04:42:37,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 in 170 msec 2024-11-19T04:42:37,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-19T04:42:37,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, ASSIGN in 325 msec 2024-11-19T04:42:37,023 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T04:42:37,023 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991357023"}]},"ts":"1731991357023"} 2024-11-19T04:42:37,024 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T04:42:37,025 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T04:42:37,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-19T04:42:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-19T04:42:38,027 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:38,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,029 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@290948e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4188df36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,034 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:42:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:42:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@656a2a6e 2024-11-19T04:42:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:42:38,035 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:42:38,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,036 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d60d042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,037 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf759c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192056d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,040 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1a658c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,041 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41661f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,043 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2cb5bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619625e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,045 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ed1589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,047 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff33fe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176b21d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1512ddb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,050 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6e89a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,052 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf84a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56d2e1d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56693319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8bbe37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005e180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da97ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,059 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:42:38,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:42:38,059 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:42:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:42:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:42:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e1ad7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:42:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:42:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:38,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:42:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fda48f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:42:38,061 DEBUG [Thread-2093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,061 DEBUG [Thread-2082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,061 DEBUG [Thread-2086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,061 DEBUG [Thread-2097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,062 DEBUG [Thread-2095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,063 DEBUG [Thread-2078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,063 DEBUG [Thread-2084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,066 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T04:42:38,067 DEBUG [Thread-2089 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,068 DEBUG [Thread-2089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,069 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:38,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,069 DEBUG [Thread-2080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,070 DEBUG [Thread-2091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:42:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,071 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:38,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:42:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:42:38,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:42:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:38,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2] 2024-11-19T04:42:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:38,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:38,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9f558aa66d664996a882087e04d5841a is 50, key is test_row_0/A:col10/1731991358072/Put/seqid=0 2024-11-19T04:42:38,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742291_1467 (size=14341) 2024-11-19T04:42:38,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731991358091, max timestamp=1731991358112Num uniques 859; 2024-11-19T04:42:38,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-19T04:42:38,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991358091, max timestamp=1731991358112Num uniques 859; 2024-11-19T04:42:38,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:38,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991358091, max timestamp=1731991358112Num uniques 859; 2024-11-19T04:42:38,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:38,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991418139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991418143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991418144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991418145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991418146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:38,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991418257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991418266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991418267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991418267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991418267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991418476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991418487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991418487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991418487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991418487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9f558aa66d664996a882087e04d5841a 2024-11-19T04:42:38,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/921ecabf240d4ac1bf5573f03ff263df is 50, key is test_row_0/B:col10/1731991358072/Put/seqid=0 2024-11-19T04:42:38,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742292_1468 (size=12001) 2024-11-19T04:42:38,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:38,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991418787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991418806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991418807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991418807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991418807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:38,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/921ecabf240d4ac1bf5573f03ff263df 2024-11-19T04:42:38,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/18da0ec8d49643e7a28725d1f601d632 is 50, key is test_row_0/C:col10/1731991358072/Put/seqid=0 2024-11-19T04:42:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742293_1469 (size=12001) 2024-11-19T04:42:38,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:38,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:38,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:38,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:39,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:39,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:39,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991419307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:39,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991419327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:39,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991419327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:39,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991419327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:39,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991419327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:39,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:39,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/18da0ec8d49643e7a28725d1f601d632 2024-11-19T04:42:39,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9f558aa66d664996a882087e04d5841a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9f558aa66d664996a882087e04d5841a 2024-11-19T04:42:39,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9f558aa66d664996a882087e04d5841a, entries=200, sequenceid=11, filesize=14.0 K 2024-11-19T04:42:39,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/921ecabf240d4ac1bf5573f03ff263df as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/921ecabf240d4ac1bf5573f03ff263df 2024-11-19T04:42:39,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/921ecabf240d4ac1bf5573f03ff263df, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T04:42:39,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/18da0ec8d49643e7a28725d1f601d632 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/18da0ec8d49643e7a28725d1f601d632 2024-11-19T04:42:39,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/18da0ec8d49643e7a28725d1f601d632, entries=150, sequenceid=11, filesize=11.7 K 2024-11-19T04:42:39,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for 728b08bae7a306a40f52d977dcb73a88 in 1293ms, sequenceid=11, compaction requested=false 2024-11-19T04:42:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:39,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:39,450 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=138.65 KB heapSize=381.08 KB 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:39,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9201bf6dc4894287bb07a063fd842407 is 50, key is test_row_0/A:col10/1731991358145/Put/seqid=0 2024-11-19T04:42:39,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742294_1470 (size=12001) 2024-11-19T04:42:39,859 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9201bf6dc4894287bb07a063fd842407 2024-11-19T04:42:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b74c85f57a5d414a9400f06a2633ae87 is 50, key is test_row_0/B:col10/1731991358145/Put/seqid=0 2024-11-19T04:42:39,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742295_1471 (size=12001) 2024-11-19T04:42:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:40,270 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b74c85f57a5d414a9400f06a2633ae87 2024-11-19T04:42:40,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/45433f638a0d4ca48eb718ebf3a691e6 is 50, key is test_row_0/C:col10/1731991358145/Put/seqid=0 2024-11-19T04:42:40,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742296_1472 (size=12001) 2024-11-19T04:42:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:40,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991420353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991420357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991420357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,367 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,367 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991420360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991420366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991420477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991420477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991420487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,681 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/45433f638a0d4ca48eb718ebf3a691e6 2024-11-19T04:42:40,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9201bf6dc4894287bb07a063fd842407 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9201bf6dc4894287bb07a063fd842407 2024-11-19T04:42:40,688 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9201bf6dc4894287bb07a063fd842407, entries=150, sequenceid=45, filesize=11.7 K 2024-11-19T04:42:40,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b74c85f57a5d414a9400f06a2633ae87 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b74c85f57a5d414a9400f06a2633ae87 2024-11-19T04:42:40,692 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b74c85f57a5d414a9400f06a2633ae87, entries=150, sequenceid=45, filesize=11.7 K 2024-11-19T04:42:40,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/45433f638a0d4ca48eb718ebf3a691e6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/45433f638a0d4ca48eb718ebf3a691e6 2024-11-19T04:42:40,696 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/45433f638a0d4ca48eb718ebf3a691e6, entries=150, sequenceid=45, filesize=11.7 K 2024-11-19T04:42:40,696 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~381.03 KB/390176, currentSize=46.96 KB/48090 for 728b08bae7a306a40f52d977dcb73a88 in 1246ms, sequenceid=45, compaction requested=false 2024-11-19T04:42:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:40,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T04:42:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T04:42:40,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T04:42:40,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6270 sec 2024-11-19T04:42:40,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6360 sec 2024-11-19T04:42:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:40,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:42:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:40,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:40,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/5a351085b4a640bfac58f0d914cbbd3d is 50, key is test_row_0/A:col10/1731991360358/Put/seqid=0 2024-11-19T04:42:40,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742297_1473 (size=14341) 2024-11-19T04:42:40,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/5a351085b4a640bfac58f0d914cbbd3d 2024-11-19T04:42:40,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/e8c427edff3748fda9c99e7af646b13b is 50, key is test_row_0/B:col10/1731991360358/Put/seqid=0 2024-11-19T04:42:40,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742298_1474 (size=12001) 2024-11-19T04:42:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991420752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991360741, max timestamp=1731991360756Num uniques 859; 2024-11-19T04:42:40,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991360741, max timestamp=1731991360756Num uniques 859; 2024-11-19T04:42:40,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991360741, max timestamp=1731991360756Num uniques 859; 2024-11-19T04:42:40,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:40,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:40,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991420841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991420843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991420867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,941 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T04:42:40,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991420967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991420967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991421086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/e8c427edff3748fda9c99e7af646b13b 2024-11-19T04:42:41,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/2db574467fbf47a7a1c978a5e7cd5832 is 50, key is test_row_0/C:col10/1731991360358/Put/seqid=0 2024-11-19T04:42:41,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742299_1475 (size=12001) 2024-11-19T04:42:41,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:41,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991421187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:41,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991421187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991421407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991421507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991421507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:41,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/2db574467fbf47a7a1c978a5e7cd5832 2024-11-19T04:42:41,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/5a351085b4a640bfac58f0d914cbbd3d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5a351085b4a640bfac58f0d914cbbd3d 2024-11-19T04:42:41,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5a351085b4a640bfac58f0d914cbbd3d, entries=200, sequenceid=56, filesize=14.0 K 2024-11-19T04:42:41,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/e8c427edff3748fda9c99e7af646b13b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e8c427edff3748fda9c99e7af646b13b 2024-11-19T04:42:41,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e8c427edff3748fda9c99e7af646b13b, entries=150, sequenceid=56, filesize=11.7 K 2024-11-19T04:42:41,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/2db574467fbf47a7a1c978a5e7cd5832 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2db574467fbf47a7a1c978a5e7cd5832 2024-11-19T04:42:41,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2db574467fbf47a7a1c978a5e7cd5832, entries=150, sequenceid=56, filesize=11.7 K 2024-11-19T04:42:41,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 728b08bae7a306a40f52d977dcb73a88 in 849ms, sequenceid=56, compaction requested=true 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:41,551 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:41,551 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:41,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:41,552 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:41,552 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:41,552 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:41,552 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:41,552 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:41,552 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:41,552 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9f558aa66d664996a882087e04d5841a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9201bf6dc4894287bb07a063fd842407, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5a351085b4a640bfac58f0d914cbbd3d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=39.7 K 2024-11-19T04:42:41,552 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/921ecabf240d4ac1bf5573f03ff263df, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b74c85f57a5d414a9400f06a2633ae87, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e8c427edff3748fda9c99e7af646b13b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=35.2 K 2024-11-19T04:42:41,552 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f558aa66d664996a882087e04d5841a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731991358071 2024-11-19T04:42:41,552 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 921ecabf240d4ac1bf5573f03ff263df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731991358071 2024-11-19T04:42:41,553 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b74c85f57a5d414a9400f06a2633ae87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731991358142 2024-11-19T04:42:41,553 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9201bf6dc4894287bb07a063fd842407, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731991358142 2024-11-19T04:42:41,553 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a351085b4a640bfac58f0d914cbbd3d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731991360349 2024-11-19T04:42:41,553 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e8c427edff3748fda9c99e7af646b13b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731991360358 2024-11-19T04:42:41,559 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:41,559 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/bbafa11223724db08fa8d69642d6632f is 50, key is test_row_0/B:col10/1731991360358/Put/seqid=0 2024-11-19T04:42:41,560 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:41,560 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d6fb23434d614daa840a502ff88e4581 is 50, key is test_row_0/A:col10/1731991360358/Put/seqid=0 2024-11-19T04:42:41,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742300_1476 (size=12104) 2024-11-19T04:42:41,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742301_1477 (size=12104) 2024-11-19T04:42:41,567 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d6fb23434d614daa840a502ff88e4581 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d6fb23434d614daa840a502ff88e4581 2024-11-19T04:42:41,570 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into d6fb23434d614daa840a502ff88e4581(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:41,570 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:41,570 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991361551; duration=0sec 2024-11-19T04:42:41,570 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:41,570 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:41,570 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:41,571 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:41,571 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:41,571 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:41,571 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/18da0ec8d49643e7a28725d1f601d632, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/45433f638a0d4ca48eb718ebf3a691e6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2db574467fbf47a7a1c978a5e7cd5832] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=35.2 K 2024-11-19T04:42:41,571 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18da0ec8d49643e7a28725d1f601d632, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731991358071 2024-11-19T04:42:41,572 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45433f638a0d4ca48eb718ebf3a691e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731991358142 2024-11-19T04:42:41,572 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db574467fbf47a7a1c978a5e7cd5832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731991360358 2024-11-19T04:42:41,580 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:41,581 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/580cf2e51c80412ab798998afd2e837a is 50, key is test_row_0/C:col10/1731991360358/Put/seqid=0 2024-11-19T04:42:41,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742302_1478 (size=12104) 2024-11-19T04:42:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:41,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T04:42:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:41,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d5156b475a1046c1865fd7e610ba230c is 50, key is test_row_0/A:col10/1731991360841/Put/seqid=0 2024-11-19T04:42:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742303_1479 (size=14341) 2024-11-19T04:42:41,967 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/bbafa11223724db08fa8d69642d6632f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/bbafa11223724db08fa8d69642d6632f 2024-11-19T04:42:41,971 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into bbafa11223724db08fa8d69642d6632f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:41,971 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:41,971 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991361551; duration=0sec 2024-11-19T04:42:41,971 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:41,971 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:41,987 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/580cf2e51c80412ab798998afd2e837a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/580cf2e51c80412ab798998afd2e837a 2024-11-19T04:42:41,991 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into 580cf2e51c80412ab798998afd2e837a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:41,991 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:41,991 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=13, startTime=1731991361551; duration=0sec 2024-11-19T04:42:41,991 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:41,991 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:42,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991421999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991422027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991422027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991422117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T04:42:42,227 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:42,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T04:42:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T04:42:42,229 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:42,230 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:42,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T04:42:42,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991422337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d5156b475a1046c1865fd7e610ba230c 2024-11-19T04:42:42,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/180278dd22764adb94c165a8f18cb8a3 is 50, key is test_row_0/B:col10/1731991360841/Put/seqid=0 2024-11-19T04:42:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742304_1480 (size=12001) 2024-11-19T04:42:42,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/180278dd22764adb94c165a8f18cb8a3 2024-11-19T04:42:42,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/fc31e1fe7d9a4c519b1fc889c3ab8655 is 50, key is test_row_0/C:col10/1731991360841/Put/seqid=0 2024-11-19T04:42:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742305_1481 (size=12001) 2024-11-19T04:42:42,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T04:42:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991422387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991422387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,391 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,391 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T04:42:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T04:42:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991422647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:42,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T04:42:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:42,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:42,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/fc31e1fe7d9a4c519b1fc889c3ab8655 2024-11-19T04:42:42,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d5156b475a1046c1865fd7e610ba230c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d5156b475a1046c1865fd7e610ba230c 2024-11-19T04:42:42,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d5156b475a1046c1865fd7e610ba230c, entries=200, sequenceid=93, filesize=14.0 K 2024-11-19T04:42:42,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/180278dd22764adb94c165a8f18cb8a3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/180278dd22764adb94c165a8f18cb8a3 2024-11-19T04:42:42,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/180278dd22764adb94c165a8f18cb8a3, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T04:42:42,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/fc31e1fe7d9a4c519b1fc889c3ab8655 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/fc31e1fe7d9a4c519b1fc889c3ab8655 2024-11-19T04:42:42,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/fc31e1fe7d9a4c519b1fc889c3ab8655, entries=150, sequenceid=93, filesize=11.7 K 2024-11-19T04:42:42,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 728b08bae7a306a40f52d977dcb73a88 in 854ms, sequenceid=93, compaction requested=false 2024-11-19T04:42:42,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:42,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T04:42:42,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:42,841 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:42,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/5fa5ce18033146e3b9edc97d8d145085 is 50, key is test_row_0/A:col10/1731991361983/Put/seqid=0 2024-11-19T04:42:42,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742306_1482 (size=12001) 2024-11-19T04:42:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T04:42:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:43,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:43,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991363081, max timestamp=1731991363121Num uniques 859; 2024-11-19T04:42:43,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991363081, max timestamp=1731991363121Num uniques 859; 2024-11-19T04:42:43,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:42:43,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:42:43,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991363081, max timestamp=1731991363121Num uniques 859; 2024-11-19T04:42:43,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:42:43,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991423185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991423191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991423198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,250 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/5fa5ce18033146e3b9edc97d8d145085 2024-11-19T04:42:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b0e422c1d2894bc1abd7b6c0c53a4ea0 is 50, key is test_row_0/B:col10/1731991361983/Put/seqid=0 2024-11-19T04:42:43,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742307_1483 (size=12001) 2024-11-19T04:42:43,263 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b0e422c1d2894bc1abd7b6c0c53a4ea0 2024-11-19T04:42:43,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/8b2b60cd8e0a43e0b58da08808656a98 is 50, key is test_row_0/C:col10/1731991361983/Put/seqid=0 2024-11-19T04:42:43,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742308_1484 (size=12001) 2024-11-19T04:42:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991423307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991423307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991423307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T04:42:43,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991423527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991423527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991423527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,680 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/8b2b60cd8e0a43e0b58da08808656a98 2024-11-19T04:42:43,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/5fa5ce18033146e3b9edc97d8d145085 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5fa5ce18033146e3b9edc97d8d145085 2024-11-19T04:42:43,688 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5fa5ce18033146e3b9edc97d8d145085, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T04:42:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b0e422c1d2894bc1abd7b6c0c53a4ea0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b0e422c1d2894bc1abd7b6c0c53a4ea0 2024-11-19T04:42:43,692 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b0e422c1d2894bc1abd7b6c0c53a4ea0, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T04:42:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/8b2b60cd8e0a43e0b58da08808656a98 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/8b2b60cd8e0a43e0b58da08808656a98 2024-11-19T04:42:43,696 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/8b2b60cd8e0a43e0b58da08808656a98, entries=150, sequenceid=105, filesize=11.7 K 2024-11-19T04:42:43,697 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 728b08bae7a306a40f52d977dcb73a88 in 856ms, sequenceid=105, compaction requested=true 2024-11-19T04:42:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T04:42:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T04:42:43,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T04:42:43,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-19T04:42:43,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4730 sec 2024-11-19T04:42:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:43,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-19T04:42:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:43,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:43,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b029dd4c3e6444fcbfce59f7a5ec9179 is 50, key is test_row_0/A:col10/1731991363852/Put/seqid=0 2024-11-19T04:42:43,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742309_1485 (size=14541) 2024-11-19T04:42:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991423873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991423873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991423873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991423997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991423997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991423997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991424217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991424217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991424217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b029dd4c3e6444fcbfce59f7a5ec9179 2024-11-19T04:42:44,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/ba6e15a7c7e44865943df9d4f18a30e7 is 50, key is test_row_0/B:col10/1731991363852/Put/seqid=0 2024-11-19T04:42:44,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742310_1486 (size=12151) 2024-11-19T04:42:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T04:42:44,377 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:44,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-19T04:42:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T04:42:44,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:44,380 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:44,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T04:42:44,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T04:42:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991424537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991424537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991424537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:44,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/ba6e15a7c7e44865943df9d4f18a30e7 2024-11-19T04:42:44,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T04:42:44,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:44,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:44,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:44,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/dd049ba7cb9c4c338b58493223cd41ef is 50, key is test_row_0/C:col10/1731991363852/Put/seqid=0 2024-11-19T04:42:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:44,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742311_1487 (size=12151) 2024-11-19T04:42:44,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/dd049ba7cb9c4c338b58493223cd41ef 2024-11-19T04:42:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T04:42:44,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b029dd4c3e6444fcbfce59f7a5ec9179 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b029dd4c3e6444fcbfce59f7a5ec9179 2024-11-19T04:42:44,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b029dd4c3e6444fcbfce59f7a5ec9179, entries=200, sequenceid=141, filesize=14.2 K 2024-11-19T04:42:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/ba6e15a7c7e44865943df9d4f18a30e7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/ba6e15a7c7e44865943df9d4f18a30e7 2024-11-19T04:42:44,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/ba6e15a7c7e44865943df9d4f18a30e7, entries=150, sequenceid=141, filesize=11.9 K 2024-11-19T04:42:44,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/dd049ba7cb9c4c338b58493223cd41ef as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/dd049ba7cb9c4c338b58493223cd41ef 2024-11-19T04:42:44,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/dd049ba7cb9c4c338b58493223cd41ef, entries=150, sequenceid=141, filesize=11.9 K 2024-11-19T04:42:44,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 728b08bae7a306a40f52d977dcb73a88 in 856ms, sequenceid=141, compaction requested=true 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:44,709 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:44,709 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:44,710 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:44,710 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:44,710 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:44,710 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:44,710 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,710 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,710 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/bbafa11223724db08fa8d69642d6632f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/180278dd22764adb94c165a8f18cb8a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b0e422c1d2894bc1abd7b6c0c53a4ea0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/ba6e15a7c7e44865943df9d4f18a30e7] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=47.1 K 2024-11-19T04:42:44,710 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d6fb23434d614daa840a502ff88e4581, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d5156b475a1046c1865fd7e610ba230c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5fa5ce18033146e3b9edc97d8d145085, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b029dd4c3e6444fcbfce59f7a5ec9179] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=51.7 K 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting bbafa11223724db08fa8d69642d6632f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731991360358 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6fb23434d614daa840a502ff88e4581, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731991360358 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 180278dd22764adb94c165a8f18cb8a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731991360812 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5156b475a1046c1865fd7e610ba230c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731991360812 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa5ce18033146e3b9edc97d8d145085, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731991361980 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e422c1d2894bc1abd7b6c0c53a4ea0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731991361980 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b029dd4c3e6444fcbfce59f7a5ec9179, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731991363184 2024-11-19T04:42:44,711 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ba6e15a7c7e44865943df9d4f18a30e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731991363184 2024-11-19T04:42:44,720 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:44,721 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/c0b64bd4c2be48fa9c97c1aa955e0278 is 50, key is test_row_0/B:col10/1731991363852/Put/seqid=0 2024-11-19T04:42:44,721 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:44,721 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/13dd39ab7c764a32995df5deead98a0a is 50, key is test_row_0/A:col10/1731991363852/Put/seqid=0 2024-11-19T04:42:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742312_1488 (size=12391) 2024-11-19T04:42:44,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742313_1489 (size=12391) 2024-11-19T04:42:44,739 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/13dd39ab7c764a32995df5deead98a0a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/13dd39ab7c764a32995df5deead98a0a 2024-11-19T04:42:44,743 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into 13dd39ab7c764a32995df5deead98a0a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:44,743 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:44,743 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=12, startTime=1731991364709; duration=0sec 2024-11-19T04:42:44,743 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:44,743 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:44,743 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:44,744 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:44,744 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:44,744 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,744 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/580cf2e51c80412ab798998afd2e837a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/fc31e1fe7d9a4c519b1fc889c3ab8655, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/8b2b60cd8e0a43e0b58da08808656a98, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/dd049ba7cb9c4c338b58493223cd41ef] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=47.1 K 2024-11-19T04:42:44,744 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580cf2e51c80412ab798998afd2e837a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731991360358 2024-11-19T04:42:44,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc31e1fe7d9a4c519b1fc889c3ab8655, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731991360812 2024-11-19T04:42:44,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b2b60cd8e0a43e0b58da08808656a98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731991361980 2024-11-19T04:42:44,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd049ba7cb9c4c338b58493223cd41ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731991363184 2024-11-19T04:42:44,755 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:44,756 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/1e62e9c1f88c4891bd82d1d3a120f72a is 50, key is test_row_0/C:col10/1731991363852/Put/seqid=0 2024-11-19T04:42:44,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742314_1490 (size=12391) 2024-11-19T04:42:44,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:44,838 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:44,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:44,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b05be543a9ee44babae280661c56de49 is 50, key is test_row_0/A:col10/1731991363870/Put/seqid=0 2024-11-19T04:42:44,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742315_1491 (size=9757) 2024-11-19T04:42:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T04:42:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:45,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:45,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991365088, max timestamp=1731991365109Num uniques 859; 2024-11-19T04:42:45,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:42:45,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991365088, max timestamp=1731991365109Num uniques 859; 2024-11-19T04:42:45,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:42:45,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731991365088, max timestamp=1731991365109Num uniques 859; 2024-11-19T04:42:45,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-19T04:42:45,134 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/c0b64bd4c2be48fa9c97c1aa955e0278 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c0b64bd4c2be48fa9c97c1aa955e0278 2024-11-19T04:42:45,138 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into c0b64bd4c2be48fa9c97c1aa955e0278(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:45,138 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:45,138 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=12, startTime=1731991364709; duration=0sec 2024-11-19T04:42:45,138 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:45,138 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:45,166 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/1e62e9c1f88c4891bd82d1d3a120f72a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1e62e9c1f88c4891bd82d1d3a120f72a 2024-11-19T04:42:45,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991425156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991425165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into 1e62e9c1f88c4891bd82d1d3a120f72a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:45,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:45,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=12, startTime=1731991364709; duration=0sec 2024-11-19T04:42:45,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:45,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991425167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,247 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b05be543a9ee44babae280661c56de49 2024-11-19T04:42:45,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/519f748bda424d3b9096de3cc67e1a8c is 50, key is test_row_0/B:col10/1731991363870/Put/seqid=0 2024-11-19T04:42:45,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742316_1492 (size=9757) 2024-11-19T04:42:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991425277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991425277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991425287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991425497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991425497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991425507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T04:42:45,658 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/519f748bda424d3b9096de3cc67e1a8c 2024-11-19T04:42:45,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/c420beab3de04ee08df061ffd243fe22 is 50, key is test_row_0/C:col10/1731991363870/Put/seqid=0 2024-11-19T04:42:45,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742317_1493 (size=9757) 2024-11-19T04:42:45,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991425807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991425807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:45,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991425816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,069 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/c420beab3de04ee08df061ffd243fe22 2024-11-19T04:42:46,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b05be543a9ee44babae280661c56de49 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b05be543a9ee44babae280661c56de49 2024-11-19T04:42:46,076 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b05be543a9ee44babae280661c56de49, entries=100, sequenceid=151, filesize=9.5 K 2024-11-19T04:42:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/519f748bda424d3b9096de3cc67e1a8c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/519f748bda424d3b9096de3cc67e1a8c 2024-11-19T04:42:46,080 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/519f748bda424d3b9096de3cc67e1a8c, entries=100, sequenceid=151, filesize=9.5 K 2024-11-19T04:42:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/c420beab3de04ee08df061ffd243fe22 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/c420beab3de04ee08df061ffd243fe22 2024-11-19T04:42:46,083 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/c420beab3de04ee08df061ffd243fe22, entries=100, sequenceid=151, filesize=9.5 K 2024-11-19T04:42:46,084 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 728b08bae7a306a40f52d977dcb73a88 in 1246ms, sequenceid=151, compaction requested=false 2024-11-19T04:42:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-19T04:42:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-19T04:42:46,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T04:42:46,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-19T04:42:46,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7100 sec 2024-11-19T04:42:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:46,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-19T04:42:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:46,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:46,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/2b526ddd63bf4d2ca0e282345659c25b is 50, key is test_row_0/A:col10/1731991365123/Put/seqid=0 2024-11-19T04:42:46,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742318_1494 (size=14541) 2024-11-19T04:42:46,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/2b526ddd63bf4d2ca0e282345659c25b 2024-11-19T04:42:46,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b217bea760ac47c9bafbdd21c64eedb3 is 50, key is test_row_0/B:col10/1731991365123/Put/seqid=0 2024-11-19T04:42:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742319_1495 (size=12151) 2024-11-19T04:42:46,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b217bea760ac47c9bafbdd21c64eedb3 2024-11-19T04:42:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991426350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991426350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/6b3d5927fbfe48ee99e2b4a1706c6aaa is 50, key is test_row_0/C:col10/1731991365123/Put/seqid=0 2024-11-19T04:42:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991426351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742320_1496 (size=12151) 2024-11-19T04:42:46,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991426407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,418 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991426417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,425 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991426467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991426467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991426477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T04:42:46,527 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:46,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-19T04:42:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T04:42:46,529 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:46,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:46,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T04:42:46,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T04:42:46,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:46,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:46,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:46,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:46,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991426676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991426687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991426697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:46,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:46,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/6b3d5927fbfe48ee99e2b4a1706c6aaa 2024-11-19T04:42:46,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/2b526ddd63bf4d2ca0e282345659c25b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/2b526ddd63bf4d2ca0e282345659c25b 2024-11-19T04:42:46,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/2b526ddd63bf4d2ca0e282345659c25b, entries=200, sequenceid=190, filesize=14.2 K 2024-11-19T04:42:46,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b217bea760ac47c9bafbdd21c64eedb3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b217bea760ac47c9bafbdd21c64eedb3 2024-11-19T04:42:46,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b217bea760ac47c9bafbdd21c64eedb3, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T04:42:46,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/6b3d5927fbfe48ee99e2b4a1706c6aaa as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/6b3d5927fbfe48ee99e2b4a1706c6aaa 2024-11-19T04:42:46,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/6b3d5927fbfe48ee99e2b4a1706c6aaa, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T04:42:46,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 728b08bae7a306a40f52d977dcb73a88 in 447ms, sequenceid=190, compaction requested=true 2024-11-19T04:42:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:46,782 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:46,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:46,783 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:46,783 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:46,783 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:46,783 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:46,783 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/13dd39ab7c764a32995df5deead98a0a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b05be543a9ee44babae280661c56de49, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/2b526ddd63bf4d2ca0e282345659c25b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=35.8 K 2024-11-19T04:42:46,784 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:46,784 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13dd39ab7c764a32995df5deead98a0a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731991363184 2024-11-19T04:42:46,784 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:46,784 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:46,784 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c0b64bd4c2be48fa9c97c1aa955e0278, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/519f748bda424d3b9096de3cc67e1a8c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b217bea760ac47c9bafbdd21c64eedb3] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=33.5 K 2024-11-19T04:42:46,784 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b05be543a9ee44babae280661c56de49, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731991363870 2024-11-19T04:42:46,785 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b64bd4c2be48fa9c97c1aa955e0278, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731991363184 2024-11-19T04:42:46,785 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b526ddd63bf4d2ca0e282345659c25b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991365123 2024-11-19T04:42:46,785 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 519f748bda424d3b9096de3cc67e1a8c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731991363870 2024-11-19T04:42:46,785 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b217bea760ac47c9bafbdd21c64eedb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991365123 2024-11-19T04:42:46,791 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:46,791 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:46,792 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/2ba94178c1c24468a72c64d681d12f1d is 50, key is test_row_0/B:col10/1731991365123/Put/seqid=0 2024-11-19T04:42:46,792 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/1a5d83f495c2451bbc3d48fc7e9d5e2f is 50, key is test_row_0/A:col10/1731991365123/Put/seqid=0 2024-11-19T04:42:46,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742321_1497 (size=12493) 2024-11-19T04:42:46,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742322_1498 (size=12493) 2024-11-19T04:42:46,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-19T04:42:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:46,835 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-19T04:42:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/cce06a0459304cbaad70b127c5ed2c83 is 50, key is test_row_0/A:col10/1731991366348/Put/seqid=0 2024-11-19T04:42:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T04:42:46,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742323_1499 (size=12151) 2024-11-19T04:42:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:47,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:47,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991367042, max timestamp=1731991367072Num uniques 859; 2024-11-19T04:42:47,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991367042, max timestamp=1731991367072Num uniques 859; 2024-11-19T04:42:47,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:47,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:47,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731991367042, max timestamp=1731991367072Num uniques 859; 2024-11-19T04:42:47,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-19T04:42:47,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991427137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991427138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991427144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T04:42:47,204 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/2ba94178c1c24468a72c64d681d12f1d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2ba94178c1c24468a72c64d681d12f1d 2024-11-19T04:42:47,205 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/1a5d83f495c2451bbc3d48fc7e9d5e2f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/1a5d83f495c2451bbc3d48fc7e9d5e2f 2024-11-19T04:42:47,208 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into 2ba94178c1c24468a72c64d681d12f1d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:47,208 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:47,208 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991366782; duration=0sec 2024-11-19T04:42:47,208 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:47,208 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:47,209 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:47,209 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into 1a5d83f495c2451bbc3d48fc7e9d5e2f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:47,209 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:47,209 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991366782; duration=0sec 2024-11-19T04:42:47,209 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:47,209 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:47,210 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:47,210 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:47,210 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:47,210 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1e62e9c1f88c4891bd82d1d3a120f72a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/c420beab3de04ee08df061ffd243fe22, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/6b3d5927fbfe48ee99e2b4a1706c6aaa] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=33.5 K 2024-11-19T04:42:47,210 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e62e9c1f88c4891bd82d1d3a120f72a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731991363184 2024-11-19T04:42:47,210 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c420beab3de04ee08df061ffd243fe22, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731991363870 2024-11-19T04:42:47,211 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3d5927fbfe48ee99e2b4a1706c6aaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991365123 2024-11-19T04:42:47,217 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:47,217 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/d4853bfc98a1437786e7b541d7ff66c8 is 50, key is test_row_0/C:col10/1731991365123/Put/seqid=0 2024-11-19T04:42:47,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742324_1500 (size=12493) 2024-11-19T04:42:47,248 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/cce06a0459304cbaad70b127c5ed2c83 2024-11-19T04:42:47,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3 is 50, key is test_row_0/B:col10/1731991366348/Put/seqid=0 2024-11-19T04:42:47,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742325_1501 (size=12151) 2024-11-19T04:42:47,260 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3 2024-11-19T04:42:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/de22f8d757184fdb9aab266ee75c646e is 50, key is test_row_0/C:col10/1731991366348/Put/seqid=0 2024-11-19T04:42:47,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991427257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991427257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991427257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742326_1502 (size=12151) 2024-11-19T04:42:47,270 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/de22f8d757184fdb9aab266ee75c646e 2024-11-19T04:42:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/cce06a0459304cbaad70b127c5ed2c83 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/cce06a0459304cbaad70b127c5ed2c83 2024-11-19T04:42:47,282 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/cce06a0459304cbaad70b127c5ed2c83, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T04:42:47,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3 2024-11-19T04:42:47,288 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T04:42:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/de22f8d757184fdb9aab266ee75c646e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/de22f8d757184fdb9aab266ee75c646e 2024-11-19T04:42:47,293 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/de22f8d757184fdb9aab266ee75c646e, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T04:42:47,293 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 728b08bae7a306a40f52d977dcb73a88 in 458ms, sequenceid=198, compaction requested=false 2024-11-19T04:42:47,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:47,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:47,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-19T04:42:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-19T04:42:47,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T04:42:47,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 765 msec 2024-11-19T04:42:47,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 769 msec 2024-11-19T04:42:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:47,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-19T04:42:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T04:42:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T04:42:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-19T04:42:47,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b1f8a1808ab846789c00eb7bc47bb8c9 is 50, key is test_row_0/A:col10/1731991367481/Put/seqid=0 2024-11-19T04:42:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742327_1503 (size=14541) 2024-11-19T04:42:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991427487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991427499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991427500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991427607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991427607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991427607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,625 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/d4853bfc98a1437786e7b541d7ff66c8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/d4853bfc98a1437786e7b541d7ff66c8 2024-11-19T04:42:47,629 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into d4853bfc98a1437786e7b541d7ff66c8(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:47,629 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:47,629 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=13, startTime=1731991366782; duration=0sec 2024-11-19T04:42:47,630 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:47,630 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-19T04:42:47,667 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:47,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-19T04:42:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T04:42:47,669 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:47,670 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:47,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T04:42:47,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:47,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:47,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:47,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:47,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991427817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991427817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991427817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:47,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b1f8a1808ab846789c00eb7bc47bb8c9 2024-11-19T04:42:47,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/39ce01761f024c87af066e302c8ef749 is 50, key is test_row_0/B:col10/1731991367481/Put/seqid=0 2024-11-19T04:42:47,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742328_1504 (size=12151) 2024-11-19T04:42:47,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:47,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:47,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T04:42:48,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991428127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:48,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991428137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991428137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:48,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T04:42:48,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/39ce01761f024c87af066e302c8ef749 2024-11-19T04:42:48,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/92ee0179bd504d6f9a2d7fa7c524ce3f is 50, key is test_row_0/C:col10/1731991367481/Put/seqid=0 2024-11-19T04:42:48,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742329_1505 (size=12151) 2024-11-19T04:42:48,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:48,589 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46599 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=08a7f35e60d4,32975,1731991233019, table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T04:42:48,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991428637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991428657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:48,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991428657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:48,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/92ee0179bd504d6f9a2d7fa7c524ce3f 2024-11-19T04:42:48,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/b1f8a1808ab846789c00eb7bc47bb8c9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b1f8a1808ab846789c00eb7bc47bb8c9 2024-11-19T04:42:48,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b1f8a1808ab846789c00eb7bc47bb8c9, entries=200, sequenceid=238, filesize=14.2 K 2024-11-19T04:42:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/39ce01761f024c87af066e302c8ef749 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/39ce01761f024c87af066e302c8ef749 2024-11-19T04:42:48,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/39ce01761f024c87af066e302c8ef749, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T04:42:48,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/92ee0179bd504d6f9a2d7fa7c524ce3f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/92ee0179bd504d6f9a2d7fa7c524ce3f 2024-11-19T04:42:48,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/92ee0179bd504d6f9a2d7fa7c524ce3f, entries=150, sequenceid=238, filesize=11.9 K 2024-11-19T04:42:48,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 728b08bae7a306a40f52d977dcb73a88 in 1249ms, sequenceid=238, compaction requested=true 2024-11-19T04:42:48,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:48,732 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:48,732 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:48,736 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:48,736 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:48,736 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:48,736 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:48,736 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,736 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,736 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2ba94178c1c24468a72c64d681d12f1d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/39ce01761f024c87af066e302c8ef749] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=35.9 K 2024-11-19T04:42:48,736 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/1a5d83f495c2451bbc3d48fc7e9d5e2f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/cce06a0459304cbaad70b127c5ed2c83, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b1f8a1808ab846789c00eb7bc47bb8c9] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=38.3 K 2024-11-19T04:42:48,736 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba94178c1c24468a72c64d681d12f1d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991365123 2024-11-19T04:42:48,737 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5d83f495c2451bbc3d48fc7e9d5e2f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991365123 2024-11-19T04:42:48,737 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a49ebedd4ed84c1d9d30fcdf7c91a8b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991366347 2024-11-19T04:42:48,737 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting cce06a0459304cbaad70b127c5ed2c83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991366347 2024-11-19T04:42:48,737 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ce01761f024c87af066e302c8ef749, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731991367116 2024-11-19T04:42:48,738 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f8a1808ab846789c00eb7bc47bb8c9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731991367116 2024-11-19T04:42:48,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,740 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:48,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:48,745 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:48,746 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/22c06784260441a1a6f00fbf9507156d is 50, key is test_row_0/B:col10/1731991367481/Put/seqid=0 2024-11-19T04:42:48,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d2332bf368fd4bb7a0e10040400f5dcc is 50, key is test_row_0/A:col10/1731991367498/Put/seqid=0 2024-11-19T04:42:48,753 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:48,754 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/bb1b513843c14676864a5a8736b5e659 is 50, key is test_row_0/A:col10/1731991367481/Put/seqid=0 2024-11-19T04:42:48,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742330_1506 (size=12595) 2024-11-19T04:42:48,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742331_1507 (size=12151) 2024-11-19T04:42:48,770 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/22c06784260441a1a6f00fbf9507156d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/22c06784260441a1a6f00fbf9507156d 2024-11-19T04:42:48,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742332_1508 (size=12595) 2024-11-19T04:42:48,777 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/bb1b513843c14676864a5a8736b5e659 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/bb1b513843c14676864a5a8736b5e659 2024-11-19T04:42:48,777 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into 22c06784260441a1a6f00fbf9507156d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:48,777 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:48,777 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991368732; duration=0sec 2024-11-19T04:42:48,778 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:48,778 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:48,778 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:48,780 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:48,780 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:48,780 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:48,780 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/d4853bfc98a1437786e7b541d7ff66c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/de22f8d757184fdb9aab266ee75c646e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/92ee0179bd504d6f9a2d7fa7c524ce3f] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=35.9 K 2024-11-19T04:42:48,780 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d4853bfc98a1437786e7b541d7ff66c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991365123 2024-11-19T04:42:48,781 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting de22f8d757184fdb9aab266ee75c646e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731991366347 2024-11-19T04:42:48,781 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ee0179bd504d6f9a2d7fa7c524ce3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731991367116 2024-11-19T04:42:48,784 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into bb1b513843c14676864a5a8736b5e659(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:48,785 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:48,785 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991368731; duration=0sec 2024-11-19T04:42:48,785 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:48,785 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:48,789 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:48,789 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/ae81668a4bf34dd5981a5e1d3f1629b5 is 50, key is test_row_0/C:col10/1731991367481/Put/seqid=0 2024-11-19T04:42:48,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742333_1509 (size=12595) 2024-11-19T04:42:48,800 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/ae81668a4bf34dd5981a5e1d3f1629b5 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/ae81668a4bf34dd5981a5e1d3f1629b5 2024-11-19T04:42:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T04:42:48,808 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into ae81668a4bf34dd5981a5e1d3f1629b5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:48,808 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:48,808 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=13, startTime=1731991368732; duration=0sec 2024-11-19T04:42:48,808 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:48,808 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:49,167 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d2332bf368fd4bb7a0e10040400f5dcc 2024-11-19T04:42:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/c172f714a13b4453b7b6b3f940c3400b is 50, key is test_row_0/B:col10/1731991367498/Put/seqid=0 2024-11-19T04:42:49,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742334_1510 (size=12151) 2024-11-19T04:42:49,179 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/c172f714a13b4453b7b6b3f940c3400b 2024-11-19T04:42:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/65e9075de9484d3b8176f14fac0e0bd3 is 50, key is test_row_0/C:col10/1731991367498/Put/seqid=0 2024-11-19T04:42:49,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742335_1511 (size=12151) 2024-11-19T04:42:49,593 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/65e9075de9484d3b8176f14fac0e0bd3 2024-11-19T04:42:49,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/d2332bf368fd4bb7a0e10040400f5dcc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d2332bf368fd4bb7a0e10040400f5dcc 2024-11-19T04:42:49,601 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d2332bf368fd4bb7a0e10040400f5dcc, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T04:42:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/c172f714a13b4453b7b6b3f940c3400b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c172f714a13b4453b7b6b3f940c3400b 2024-11-19T04:42:49,605 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c172f714a13b4453b7b6b3f940c3400b, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T04:42:49,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/65e9075de9484d3b8176f14fac0e0bd3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/65e9075de9484d3b8176f14fac0e0bd3 2024-11-19T04:42:49,609 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/65e9075de9484d3b8176f14fac0e0bd3, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T04:42:49,611 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 728b08bae7a306a40f52d977dcb73a88 in 871ms, sequenceid=246, compaction requested=false 2024-11-19T04:42:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-19T04:42:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-19T04:42:49,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-19T04:42:49,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9420 sec 2024-11-19T04:42:49,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.9460 sec 2024-11-19T04:42:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:49,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:49,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:49,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:49,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:49,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:49,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/8de52865d48b4269bedda93b0a1fb8c1 is 50, key is test_row_0/A:col10/1731991369665/Put/seqid=0 2024-11-19T04:42:49,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742336_1512 (size=17081) 2024-11-19T04:42:49,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/8de52865d48b4269bedda93b0a1fb8c1 2024-11-19T04:42:49,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/833e71df945e487f83f9ee6428cf0bf2 is 50, key is test_row_0/B:col10/1731991369665/Put/seqid=0 2024-11-19T04:42:49,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742337_1513 (size=12301) 2024-11-19T04:42:49,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/833e71df945e487f83f9ee6428cf0bf2 2024-11-19T04:42:49,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/83504ccab7d54db1ab47cf29d2e3d7e6 is 50, key is test_row_0/C:col10/1731991369665/Put/seqid=0 2024-11-19T04:42:49,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742338_1514 (size=12301) 2024-11-19T04:42:49,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991369702, max timestamp=1731991369766Num uniques 859; 2024-11-19T04:42:49,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:49,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991369702, max timestamp=1731991369766Num uniques 859; 2024-11-19T04:42:49,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991369702, max timestamp=1731991369766Num uniques 859; 2024-11-19T04:42:49,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:49,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T04:42:49,817 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:49,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-19T04:42:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T04:42:49,818 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:49,819 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:49,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991429864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991429865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:49,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991429881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T04:42:49,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T04:42:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:49,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:49,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:49,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:49,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991429987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991429987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:49,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991429996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/83504ccab7d54db1ab47cf29d2e3d7e6 2024-11-19T04:42:50,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/8de52865d48b4269bedda93b0a1fb8c1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/8de52865d48b4269bedda93b0a1fb8c1 2024-11-19T04:42:50,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/8de52865d48b4269bedda93b0a1fb8c1, entries=250, sequenceid=259, filesize=16.7 K 2024-11-19T04:42:50,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/833e71df945e487f83f9ee6428cf0bf2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/833e71df945e487f83f9ee6428cf0bf2 2024-11-19T04:42:50,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/833e71df945e487f83f9ee6428cf0bf2, entries=150, sequenceid=259, filesize=12.0 K 2024-11-19T04:42:50,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/83504ccab7d54db1ab47cf29d2e3d7e6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/83504ccab7d54db1ab47cf29d2e3d7e6 2024-11-19T04:42:50,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/83504ccab7d54db1ab47cf29d2e3d7e6, entries=150, sequenceid=259, filesize=12.0 K 2024-11-19T04:42:50,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 728b08bae7a306a40f52d977dcb73a88 in 455ms, sequenceid=259, compaction requested=true 2024-11-19T04:42:50,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:50,122 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:50,122 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:50,123 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:50,123 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:50,123 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:50,123 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/bb1b513843c14676864a5a8736b5e659, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d2332bf368fd4bb7a0e10040400f5dcc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/8de52865d48b4269bedda93b0a1fb8c1] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=40.8 K 2024-11-19T04:42:50,123 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:50,123 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:50,123 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:50,123 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/22c06784260441a1a6f00fbf9507156d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c172f714a13b4453b7b6b3f940c3400b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/833e71df945e487f83f9ee6428cf0bf2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=36.2 K 2024-11-19T04:42:50,123 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c06784260441a1a6f00fbf9507156d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731991367116 2024-11-19T04:42:50,123 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1b513843c14676864a5a8736b5e659, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731991367116 2024-11-19T04:42:50,124 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2332bf368fd4bb7a0e10040400f5dcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731991367485 2024-11-19T04:42:50,124 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c172f714a13b4453b7b6b3f940c3400b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731991367485 2024-11-19T04:42:50,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-19T04:42:50,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:50,125 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-19T04:42:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:50,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:50,125 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 833e71df945e487f83f9ee6428cf0bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991369655 2024-11-19T04:42:50,125 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8de52865d48b4269bedda93b0a1fb8c1, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991369651 2024-11-19T04:42:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/797f30384e214fdba230ab4225df4449 is 50, key is test_row_0/A:col10/1731991369879/Put/seqid=0 2024-11-19T04:42:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T04:42:50,139 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:50,139 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:50,139 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b19eaeddea5142e4a7a2c20c6ff2c311 is 50, key is test_row_0/B:col10/1731991369665/Put/seqid=0 2024-11-19T04:42:50,140 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/21abf104854d42eb8853006e069d35c8 is 50, key is test_row_0/A:col10/1731991369665/Put/seqid=0 2024-11-19T04:42:50,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742339_1515 (size=12301) 2024-11-19T04:42:50,160 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/797f30384e214fdba230ab4225df4449 2024-11-19T04:42:50,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742340_1516 (size=12847) 2024-11-19T04:42:50,171 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/21abf104854d42eb8853006e069d35c8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/21abf104854d42eb8853006e069d35c8 2024-11-19T04:42:50,175 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into 21abf104854d42eb8853006e069d35c8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:50,175 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:50,175 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991370121; duration=0sec 2024-11-19T04:42:50,175 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:50,175 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:50,175 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:50,176 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:50,176 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:50,176 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:50,176 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/ae81668a4bf34dd5981a5e1d3f1629b5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/65e9075de9484d3b8176f14fac0e0bd3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/83504ccab7d54db1ab47cf29d2e3d7e6] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=36.2 K 2024-11-19T04:42:50,179 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae81668a4bf34dd5981a5e1d3f1629b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731991367116 2024-11-19T04:42:50,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742341_1517 (size=12847) 2024-11-19T04:42:50,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/4d884b485dd74479aaf2ef52457ef0b8 is 50, key is test_row_0/B:col10/1731991369879/Put/seqid=0 2024-11-19T04:42:50,179 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e9075de9484d3b8176f14fac0e0bd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731991367485 2024-11-19T04:42:50,180 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83504ccab7d54db1ab47cf29d2e3d7e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991369655 2024-11-19T04:42:50,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742342_1518 (size=12301) 2024-11-19T04:42:50,195 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:50,196 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/02f73babe88b4a3b8682fc48a75bc62b is 50, key is test_row_0/C:col10/1731991369665/Put/seqid=0 2024-11-19T04:42:50,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742343_1519 (size=12847) 2024-11-19T04:42:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991430236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991430237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991430240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991430357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991430357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T04:42:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991430567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991430567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991430567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:50,583 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/b19eaeddea5142e4a7a2c20c6ff2c311 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b19eaeddea5142e4a7a2c20c6ff2c311 2024-11-19T04:42:50,587 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into b19eaeddea5142e4a7a2c20c6ff2c311(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:50,588 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:50,588 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991370122; duration=0sec 2024-11-19T04:42:50,588 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:50,588 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:50,596 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/4d884b485dd74479aaf2ef52457ef0b8 2024-11-19T04:42:50,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/a04c4fbc3cf743ee9cca0df826cee0ed is 50, key is test_row_0/C:col10/1731991369879/Put/seqid=0 2024-11-19T04:42:50,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742344_1520 (size=12301) 2024-11-19T04:42:50,609 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/a04c4fbc3cf743ee9cca0df826cee0ed 2024-11-19T04:42:50,614 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/02f73babe88b4a3b8682fc48a75bc62b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/02f73babe88b4a3b8682fc48a75bc62b 2024-11-19T04:42:50,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/797f30384e214fdba230ab4225df4449 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/797f30384e214fdba230ab4225df4449 2024-11-19T04:42:50,620 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into 02f73babe88b4a3b8682fc48a75bc62b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:50,620 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:50,620 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=13, startTime=1731991370122; duration=0sec 2024-11-19T04:42:50,620 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:50,620 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:50,623 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/797f30384e214fdba230ab4225df4449, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T04:42:50,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/4d884b485dd74479aaf2ef52457ef0b8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/4d884b485dd74479aaf2ef52457ef0b8 2024-11-19T04:42:50,629 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/4d884b485dd74479aaf2ef52457ef0b8, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T04:42:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/a04c4fbc3cf743ee9cca0df826cee0ed as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a04c4fbc3cf743ee9cca0df826cee0ed 2024-11-19T04:42:50,635 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a04c4fbc3cf743ee9cca0df826cee0ed, entries=150, sequenceid=296, filesize=12.0 K 2024-11-19T04:42:50,635 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 728b08bae7a306a40f52d977dcb73a88 in 510ms, sequenceid=296, compaction requested=false 2024-11-19T04:42:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:50,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-19T04:42:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-19T04:42:50,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T04:42:50,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 820 msec 2024-11-19T04:42:50,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 825 msec 2024-11-19T04:42:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:50,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-19T04:42:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:50,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:50,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3a70c843f6c1426fb62f0879e58b7cf2 is 50, key is test_row_0/A:col10/1731991370234/Put/seqid=0 2024-11-19T04:42:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742345_1521 (size=14741) 2024-11-19T04:42:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-19T04:42:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991370930, max timestamp=1731991370954Num uniques 859; 2024-11-19T04:42:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:50,957 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991370930, max timestamp=1731991370954Num uniques 859; 2024-11-19T04:42:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991370930, max timestamp=1731991370954Num uniques 859; 2024-11-19T04:42:50,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:50,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-19T04:42:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:50,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:50,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:50,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:51,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991431011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991431013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991431013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:51,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:51,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991431117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991431137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991431137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:51,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:51,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:51,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3a70c843f6c1426fb62f0879e58b7cf2 2024-11-19T04:42:51,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/51261088ae594280a07dde27748dfb02 is 50, key is test_row_0/B:col10/1731991370234/Put/seqid=0 2024-11-19T04:42:51,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742346_1522 (size=12301) 2024-11-19T04:42:51,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991431327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991431357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991431357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:51,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:51,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:51,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:51,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991431647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991431677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991431677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:51,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/51261088ae594280a07dde27748dfb02 2024-11-19T04:42:51,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/67628342eeae4e3db1cc5c21cd8ee497 is 50, key is test_row_0/C:col10/1731991370234/Put/seqid=0 2024-11-19T04:42:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742347_1523 (size=12301) 2024-11-19T04:42:51,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:51,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:52,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/67628342eeae4e3db1cc5c21cd8ee497 2024-11-19T04:42:52,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3a70c843f6c1426fb62f0879e58b7cf2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a70c843f6c1426fb62f0879e58b7cf2 2024-11-19T04:42:52,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a70c843f6c1426fb62f0879e58b7cf2, entries=200, sequenceid=310, filesize=14.4 K 2024-11-19T04:42:52,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/51261088ae594280a07dde27748dfb02 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/51261088ae594280a07dde27748dfb02 2024-11-19T04:42:52,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/51261088ae594280a07dde27748dfb02, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T04:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/67628342eeae4e3db1cc5c21cd8ee497 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/67628342eeae4e3db1cc5c21cd8ee497 2024-11-19T04:42:52,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/67628342eeae4e3db1cc5c21cd8ee497, entries=150, sequenceid=310, filesize=12.0 K 2024-11-19T04:42:52,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 728b08bae7a306a40f52d977dcb73a88 in 1247ms, sequenceid=310, compaction requested=true 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:52,138 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:52,138 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:52,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:52,139 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:52,139 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:52,139 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:52,139 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:52,139 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,139 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,139 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b19eaeddea5142e4a7a2c20c6ff2c311, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/4d884b485dd74479aaf2ef52457ef0b8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/51261088ae594280a07dde27748dfb02] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=36.6 K 2024-11-19T04:42:52,139 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/21abf104854d42eb8853006e069d35c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/797f30384e214fdba230ab4225df4449, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a70c843f6c1426fb62f0879e58b7cf2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=39.0 K 2024-11-19T04:42:52,139 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b19eaeddea5142e4a7a2c20c6ff2c311, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991369655 2024-11-19T04:42:52,139 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21abf104854d42eb8853006e069d35c8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991369655 2024-11-19T04:42:52,140 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d884b485dd74479aaf2ef52457ef0b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731991369840 2024-11-19T04:42:52,140 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 797f30384e214fdba230ab4225df4449, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731991369840 2024-11-19T04:42:52,140 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 51261088ae594280a07dde27748dfb02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731991370234 2024-11-19T04:42:52,140 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a70c843f6c1426fb62f0879e58b7cf2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731991370217 2024-11-19T04:42:52,146 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:52,147 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/6dffa3a62f0e473e89d38057781b3b5a is 50, key is test_row_0/A:col10/1731991370234/Put/seqid=0 2024-11-19T04:42:52,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742348_1524 (size=12949) 2024-11-19T04:42:52,157 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:52,158 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/6ae6b76a8c94459b8d3bff89a194a7b6 is 50, key is test_row_0/B:col10/1731991370234/Put/seqid=0 2024-11-19T04:42:52,164 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/6dffa3a62f0e473e89d38057781b3b5a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/6dffa3a62f0e473e89d38057781b3b5a 2024-11-19T04:42:52,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742349_1525 (size=12949) 2024-11-19T04:42:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:52,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-19T04:42:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:52,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:52,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into 6dffa3a62f0e473e89d38057781b3b5a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:52,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:52,170 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991372138; duration=0sec 2024-11-19T04:42:52,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:52,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:52,170 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:52,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/e7f2c29dd18c4bb59beab161a540732e is 50, key is test_row_0/A:col10/1731991371011/Put/seqid=0 2024-11-19T04:42:52,173 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/6ae6b76a8c94459b8d3bff89a194a7b6 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/6ae6b76a8c94459b8d3bff89a194a7b6 2024-11-19T04:42:52,176 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:52,176 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:52,176 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,176 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/02f73babe88b4a3b8682fc48a75bc62b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a04c4fbc3cf743ee9cca0df826cee0ed, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/67628342eeae4e3db1cc5c21cd8ee497] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=36.6 K 2024-11-19T04:42:52,176 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f73babe88b4a3b8682fc48a75bc62b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731991369655 2024-11-19T04:42:52,177 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04c4fbc3cf743ee9cca0df826cee0ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731991369840 2024-11-19T04:42:52,177 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67628342eeae4e3db1cc5c21cd8ee497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731991370234 2024-11-19T04:42:52,180 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into 6ae6b76a8c94459b8d3bff89a194a7b6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:52,180 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:52,180 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991372138; duration=0sec 2024-11-19T04:42:52,180 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:52,180 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:52,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742350_1526 (size=14741) 2024-11-19T04:42:52,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/e7f2c29dd18c4bb59beab161a540732e 2024-11-19T04:42:52,186 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:52,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,186 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/82053f4a4680452a96683e5581ace930 is 50, key is test_row_0/C:col10/1731991370234/Put/seqid=0 2024-11-19T04:42:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742351_1527 (size=12949) 2024-11-19T04:42:52,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/588e5b3e7eb24d918190995fa34c9d2f is 50, key is test_row_0/B:col10/1731991371011/Put/seqid=0 2024-11-19T04:42:52,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742352_1528 (size=12301) 2024-11-19T04:42:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991432202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991432203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991432219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991432326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991432326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991432337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991432547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991432547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991432557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/588e5b3e7eb24d918190995fa34c9d2f 2024-11-19T04:42:52,596 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/82053f4a4680452a96683e5581ace930 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/82053f4a4680452a96683e5581ace930 2024-11-19T04:42:52,600 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into 82053f4a4680452a96683e5581ace930(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:52,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:52,600 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=13, startTime=1731991372138; duration=0sec 2024-11-19T04:42:52,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:52,600 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:52,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/1a3205dc09c040adbd212212bc9139ff is 50, key is test_row_0/C:col10/1731991371011/Put/seqid=0 2024-11-19T04:42:52,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742353_1529 (size=12301) 2024-11-19T04:42:52,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991432867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991432867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991432867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:52,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:52,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:52,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:53,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/1a3205dc09c040adbd212212bc9139ff 2024-11-19T04:42:53,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/e7f2c29dd18c4bb59beab161a540732e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/e7f2c29dd18c4bb59beab161a540732e 2024-11-19T04:42:53,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/e7f2c29dd18c4bb59beab161a540732e, entries=200, sequenceid=346, filesize=14.4 K 2024-11-19T04:42:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/588e5b3e7eb24d918190995fa34c9d2f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/588e5b3e7eb24d918190995fa34c9d2f 2024-11-19T04:42:53,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/588e5b3e7eb24d918190995fa34c9d2f, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T04:42:53,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/1a3205dc09c040adbd212212bc9139ff as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1a3205dc09c040adbd212212bc9139ff 2024-11-19T04:42:53,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1a3205dc09c040adbd212212bc9139ff, entries=150, sequenceid=346, filesize=12.0 K 2024-11-19T04:42:53,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 728b08bae7a306a40f52d977dcb73a88 in 853ms, sequenceid=346, compaction requested=false 2024-11-19T04:42:53,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:53,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:53,104 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9b12f72d81c24537a540726e9bbb4dec is 50, key is test_row_0/A:col10/1731991372197/Put/seqid=0 2024-11-19T04:42:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742354_1530 (size=12301) 2024-11-19T04:42:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:53,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991433444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991373432, max timestamp=1731991373449Num uniques 859; 2024-11-19T04:42:53,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:53,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991373432, max timestamp=1731991373449Num uniques 859; 2024-11-19T04:42:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731991373432, max timestamp=1731991373449Num uniques 859; 2024-11-19T04:42:53,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-19T04:42:53,513 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9b12f72d81c24537a540726e9bbb4dec 2024-11-19T04:42:53,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/36d3459a25344e3aa62eef007ac8a7c8 is 50, key is test_row_0/B:col10/1731991372197/Put/seqid=0 2024-11-19T04:42:53,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742355_1531 (size=12301) 2024-11-19T04:42:53,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991433525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991433538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991433557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991433646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991433657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991433777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991433866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991433877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:53,934 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/36d3459a25344e3aa62eef007ac8a7c8 2024-11-19T04:42:53,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/7f00d83ba55e45e798f50d8503281b10 is 50, key is test_row_0/C:col10/1731991372197/Put/seqid=0 2024-11-19T04:42:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742356_1532 (size=12301) 2024-11-19T04:42:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991434097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991434187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:54,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991434197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:54,344 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/7f00d83ba55e45e798f50d8503281b10 2024-11-19T04:42:54,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/9b12f72d81c24537a540726e9bbb4dec as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9b12f72d81c24537a540726e9bbb4dec 2024-11-19T04:42:54,352 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9b12f72d81c24537a540726e9bbb4dec, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T04:42:54,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/36d3459a25344e3aa62eef007ac8a7c8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/36d3459a25344e3aa62eef007ac8a7c8 2024-11-19T04:42:54,355 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/36d3459a25344e3aa62eef007ac8a7c8, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T04:42:54,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/7f00d83ba55e45e798f50d8503281b10 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/7f00d83ba55e45e798f50d8503281b10 2024-11-19T04:42:54,359 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/7f00d83ba55e45e798f50d8503281b10, entries=150, sequenceid=359, filesize=12.0 K 2024-11-19T04:42:54,360 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 728b08bae7a306a40f52d977dcb73a88 in 1256ms, sequenceid=359, compaction requested=true 2024-11-19T04:42:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:54,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:54,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-19T04:42:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-19T04:42:54,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T04:42:54,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4030 sec 2024-11-19T04:42:54,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 3.4080 sec 2024-11-19T04:42:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:54,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-19T04:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:54,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:54,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/a5c84449f561418e9938b1952bd683b9 is 50, key is test_row_0/A:col10/1731991373534/Put/seqid=0 2024-11-19T04:42:54,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742357_1533 (size=14741) 2024-11-19T04:42:54,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991434680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:54,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991434707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:54,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991434717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:54,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991434797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:55,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991435007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:55,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/a5c84449f561418e9938b1952bd683b9 2024-11-19T04:42:55,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/e613eb608f0841bf9cf78a769ca14a04 is 50, key is test_row_0/B:col10/1731991373534/Put/seqid=0 2024-11-19T04:42:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742358_1534 (size=12301) 2024-11-19T04:42:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T04:42:55,117 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:55,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-19T04:42:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T04:42:55,119 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:55,120 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:55,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T04:42:55,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T04:42:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991435327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:55,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:55,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T04:42:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:55,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T04:42:55,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/e613eb608f0841bf9cf78a769ca14a04 2024-11-19T04:42:55,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/a0c93601fd854672919b0d5f55260317 is 50, key is test_row_0/C:col10/1731991373534/Put/seqid=0 2024-11-19T04:42:55,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742359_1535 (size=12301) 2024-11-19T04:42:55,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T04:42:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991435717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:55,727 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:55,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T04:42:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:55,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991435737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:55,745 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T04:42:55,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/a0c93601fd854672919b0d5f55260317 2024-11-19T04:42:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991435847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:55,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/a5c84449f561418e9938b1952bd683b9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/a5c84449f561418e9938b1952bd683b9 2024-11-19T04:42:55,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/a5c84449f561418e9938b1952bd683b9, entries=200, sequenceid=396, filesize=14.4 K 2024-11-19T04:42:55,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/e613eb608f0841bf9cf78a769ca14a04 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e613eb608f0841bf9cf78a769ca14a04 2024-11-19T04:42:55,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e613eb608f0841bf9cf78a769ca14a04, entries=150, sequenceid=396, filesize=12.0 K 2024-11-19T04:42:55,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/a0c93601fd854672919b0d5f55260317 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a0c93601fd854672919b0d5f55260317 2024-11-19T04:42:55,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a0c93601fd854672919b0d5f55260317, entries=150, sequenceid=396, filesize=12.0 K 2024-11-19T04:42:55,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 728b08bae7a306a40f52d977dcb73a88 in 1246ms, sequenceid=396, compaction requested=true 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:55,872 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:55,872 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:42:55,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:55,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:55,874 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,874 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/6ae6b76a8c94459b8d3bff89a194a7b6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/588e5b3e7eb24d918190995fa34c9d2f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/36d3459a25344e3aa62eef007ac8a7c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e613eb608f0841bf9cf78a769ca14a04] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=48.7 K 2024-11-19T04:42:55,874 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:42:55,874 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:55,874 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,874 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/6dffa3a62f0e473e89d38057781b3b5a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/e7f2c29dd18c4bb59beab161a540732e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9b12f72d81c24537a540726e9bbb4dec, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/a5c84449f561418e9938b1952bd683b9] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=53.4 K 2024-11-19T04:42:55,874 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ae6b76a8c94459b8d3bff89a194a7b6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731991370234 2024-11-19T04:42:55,874 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dffa3a62f0e473e89d38057781b3b5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731991370234 2024-11-19T04:42:55,875 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 588e5b3e7eb24d918190995fa34c9d2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731991370997 2024-11-19T04:42:55,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f2c29dd18c4bb59beab161a540732e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731991370997 2024-11-19T04:42:55,875 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d3459a25344e3aa62eef007ac8a7c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731991372197 2024-11-19T04:42:55,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b12f72d81c24537a540726e9bbb4dec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731991372197 2024-11-19T04:42:55,875 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c84449f561418e9938b1952bd683b9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731991373504 2024-11-19T04:42:55,875 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting e613eb608f0841bf9cf78a769ca14a04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731991373504 2024-11-19T04:42:55,883 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:55,884 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/dd2375a4332749099415365e49d35d15 is 50, key is test_row_0/A:col10/1731991373534/Put/seqid=0 2024-11-19T04:42:55,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,884 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:55,887 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:55,887 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/0ca6b87c66104f9091c640d7b4a355c3 is 50, key is test_row_0/B:col10/1731991373534/Put/seqid=0 2024-11-19T04:42:55,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3174463819ab4501880aaa173b0ad0ae is 50, key is test_row_0/A:col10/1731991374665/Put/seqid=0 2024-11-19T04:42:55,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742361_1537 (size=13085) 2024-11-19T04:42:55,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742360_1536 (size=13085) 2024-11-19T04:42:55,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742362_1538 (size=9857) 2024-11-19T04:42:55,893 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3174463819ab4501880aaa173b0ad0ae 2024-11-19T04:42:55,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/776f4887db6c464c903bf18dff4ba5aa is 50, key is test_row_0/B:col10/1731991374665/Put/seqid=0 2024-11-19T04:42:55,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742363_1539 (size=9857) 2024-11-19T04:42:55,928 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/776f4887db6c464c903bf18dff4ba5aa 2024-11-19T04:42:55,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/2d450fde4dc7421aba40c27cd52c8ad0 is 50, key is test_row_0/C:col10/1731991374665/Put/seqid=0 2024-11-19T04:42:55,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742364_1540 (size=9857) 2024-11-19T04:42:55,969 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/2d450fde4dc7421aba40c27cd52c8ad0 2024-11-19T04:42:55,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3174463819ab4501880aaa173b0ad0ae as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3174463819ab4501880aaa173b0ad0ae 2024-11-19T04:42:55,981 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3174463819ab4501880aaa173b0ad0ae, entries=100, sequenceid=405, filesize=9.6 K 2024-11-19T04:42:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/776f4887db6c464c903bf18dff4ba5aa as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/776f4887db6c464c903bf18dff4ba5aa 2024-11-19T04:42:55,987 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/776f4887db6c464c903bf18dff4ba5aa, entries=100, sequenceid=405, filesize=9.6 K 2024-11-19T04:42:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/2d450fde4dc7421aba40c27cd52c8ad0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2d450fde4dc7421aba40c27cd52c8ad0 2024-11-19T04:42:55,995 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2d450fde4dc7421aba40c27cd52c8ad0, entries=100, sequenceid=405, filesize=9.6 K 2024-11-19T04:42:55,996 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 728b08bae7a306a40f52d977dcb73a88 in 112ms, sequenceid=405, compaction requested=true 2024-11-19T04:42:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:55,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:55,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-19T04:42:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-19T04:42:56,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T04:42:56,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 878 msec 2024-11-19T04:42:56,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 884 msec 2024-11-19T04:42:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-19T04:42:56,258 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:56,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-19T04:42:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T04:42:56,260 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:56,261 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:56,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:56,296 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/0ca6b87c66104f9091c640d7b4a355c3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/0ca6b87c66104f9091c640d7b4a355c3 2024-11-19T04:42:56,296 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/dd2375a4332749099415365e49d35d15 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/dd2375a4332749099415365e49d35d15 2024-11-19T04:42:56,300 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into 0ca6b87c66104f9091c640d7b4a355c3(size=12.8 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:56,300 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:56,300 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=12, startTime=1731991375872; duration=0sec 2024-11-19T04:42:56,300 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:56,300 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:56,300 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:42:56,300 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into dd2375a4332749099415365e49d35d15(size=12.8 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:56,300 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:56,300 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=12, startTime=1731991375872; duration=0sec 2024-11-19T04:42:56,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:56,301 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:56,302 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59709 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:42:56,302 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:42:56,302 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,302 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/82053f4a4680452a96683e5581ace930, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1a3205dc09c040adbd212212bc9139ff, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/7f00d83ba55e45e798f50d8503281b10, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a0c93601fd854672919b0d5f55260317, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2d450fde4dc7421aba40c27cd52c8ad0] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=58.3 K 2024-11-19T04:42:56,302 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 82053f4a4680452a96683e5581ace930, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731991370234 2024-11-19T04:42:56,302 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a3205dc09c040adbd212212bc9139ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731991370997 2024-11-19T04:42:56,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f00d83ba55e45e798f50d8503281b10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731991372197 2024-11-19T04:42:56,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c93601fd854672919b0d5f55260317, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731991373504 2024-11-19T04:42:56,303 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d450fde4dc7421aba40c27cd52c8ad0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731991374665 2024-11-19T04:42:56,312 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:56,313 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/be528bcbf1954dcda93ac90d998e8733 is 50, key is test_row_0/C:col10/1731991374665/Put/seqid=0 2024-11-19T04:42:56,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742365_1541 (size=13119) 2024-11-19T04:42:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T04:42:56,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-19T04:42:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-19T04:42:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-19T04:42:56,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-19T04:42:56,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-19T04:42:56,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 161 msec 2024-11-19T04:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:56,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:42:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:56,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/34b64b789e064369aec3263fb4612949 is 50, key is test_row_0/A:col10/1731991376515/Put/seqid=0 2024-11-19T04:42:56,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742366_1542 (size=17181) 2024-11-19T04:42:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-19T04:42:56,577 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:42:56,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:42:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-19T04:42:56,580 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:42:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:42:56,581 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:42:56,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:42:56,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991436588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:56,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991376577, max timestamp=1731991376593Num uniques 859; 2024-11-19T04:42:56,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:56,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991376577, max timestamp=1731991376593Num uniques 859; 2024-11-19T04:42:56,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:56,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991376577, max timestamp=1731991376593Num uniques 859; 2024-11-19T04:42:56,596 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:42:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:42:56,726 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/be528bcbf1954dcda93ac90d998e8733 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/be528bcbf1954dcda93ac90d998e8733 2024-11-19T04:42:56,730 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into be528bcbf1954dcda93ac90d998e8733(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:56,730 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:56,730 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=11, startTime=1731991375872; duration=0sec 2024-11-19T04:42:56,730 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:56,730 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:56,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991436723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:56,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:56,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991436731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:56,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:56,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991436837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:56,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991436857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991436867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:56,877 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:56,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:56,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:56,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:42:56,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/34b64b789e064369aec3263fb4612949 2024-11-19T04:42:56,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/2d86cf8f72464daf880f5f85acd0676d is 50, key is test_row_0/B:col10/1731991376515/Put/seqid=0 2024-11-19T04:42:56,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742367_1543 (size=12301) 2024-11-19T04:42:57,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991437056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991437077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:42:57,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/2d86cf8f72464daf880f5f85acd0676d 2024-11-19T04:42:57,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/466654d27a0f4858ae5a0cdcff9673a3 is 50, key is test_row_0/C:col10/1731991376515/Put/seqid=0 2024-11-19T04:42:57,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742368_1544 (size=12301) 2024-11-19T04:42:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991437377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991437397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:42:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:42:57,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/466654d27a0f4858ae5a0cdcff9673a3 2024-11-19T04:42:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35776 deadline: 1731991437747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,756 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/34b64b789e064369aec3263fb4612949 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/34b64b789e064369aec3263fb4612949 2024-11-19T04:42:57,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/34b64b789e064369aec3263fb4612949, entries=250, sequenceid=419, filesize=16.8 K 2024-11-19T04:42:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/2d86cf8f72464daf880f5f85acd0676d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2d86cf8f72464daf880f5f85acd0676d 2024-11-19T04:42:57,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2d86cf8f72464daf880f5f85acd0676d, entries=150, sequenceid=419, filesize=12.0 K 2024-11-19T04:42:57,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/466654d27a0f4858ae5a0cdcff9673a3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/466654d27a0f4858ae5a0cdcff9673a3 2024-11-19T04:42:57,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/466654d27a0f4858ae5a0cdcff9673a3, entries=150, sequenceid=419, filesize=12.0 K 2024-11-19T04:42:57,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35760 deadline: 1731991437757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 728b08bae7a306a40f52d977dcb73a88 in 1257ms, sequenceid=419, compaction requested=true 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:57,775 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:42:57,775 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:42:57,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:57,775 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:42:57,777 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:57,777 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:42:57,777 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40123 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:42:57,777 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:42:57,777 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,777 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,777 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/0ca6b87c66104f9091c640d7b4a355c3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/776f4887db6c464c903bf18dff4ba5aa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2d86cf8f72464daf880f5f85acd0676d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=34.4 K 2024-11-19T04:42:57,777 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/dd2375a4332749099415365e49d35d15, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3174463819ab4501880aaa173b0ad0ae, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/34b64b789e064369aec3263fb4612949] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=39.2 K 2024-11-19T04:42:57,778 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd2375a4332749099415365e49d35d15, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731991373504 2024-11-19T04:42:57,778 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca6b87c66104f9091c640d7b4a355c3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731991373504 2024-11-19T04:42:57,778 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3174463819ab4501880aaa173b0ad0ae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731991374665 2024-11-19T04:42:57,778 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 776f4887db6c464c903bf18dff4ba5aa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731991374665 2024-11-19T04:42:57,778 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b64b789e064369aec3263fb4612949, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731991376506 2024-11-19T04:42:57,779 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d86cf8f72464daf880f5f85acd0676d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731991376515 2024-11-19T04:42:57,785 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:57,785 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:42:57,785 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/f5967b91b75744319a2cef4aa0f2381c is 50, key is test_row_0/A:col10/1731991376515/Put/seqid=0 2024-11-19T04:42:57,785 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/774ca2d08a8d4c8cb0ad9c9d22df67ea is 50, key is test_row_0/B:col10/1731991376515/Put/seqid=0 2024-11-19T04:42:57,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742369_1545 (size=13187) 2024-11-19T04:42:57,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742370_1546 (size=13187) 2024-11-19T04:42:57,805 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/f5967b91b75744319a2cef4aa0f2381c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/f5967b91b75744319a2cef4aa0f2381c 2024-11-19T04:42:57,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-19T04:42:57,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:57,805 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T04:42:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:57,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:42:57,806 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/774ca2d08a8d4c8cb0ad9c9d22df67ea as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/774ca2d08a8d4c8cb0ad9c9d22df67ea 2024-11-19T04:42:57,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3092ef60b83342a39aec2586e6330e96 is 50, key is test_row_0/A:col10/1731991376694/Put/seqid=0 2024-11-19T04:42:57,813 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into f5967b91b75744319a2cef4aa0f2381c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:57,813 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:57,813 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991377775; duration=0sec 2024-11-19T04:42:57,813 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:42:57,813 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:42:57,813 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-19T04:42:57,813 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-19T04:42:57,814 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into 774ca2d08a8d4c8cb0ad9c9d22df67ea(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:42:57,814 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:57,814 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991377775; duration=0sec 2024-11-19T04:42:57,814 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-19T04:42:57,814 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:42:57,814 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:42:57,814 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. because compaction request was cancelled 2024-11-19T04:42:57,814 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:42:57,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742371_1547 (size=12301) 2024-11-19T04:42:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:57,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. as already flushing 2024-11-19T04:42:57,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991437942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991437943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991438057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,066 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-11-19T04:42:58,066 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:58,066 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,066 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991438067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,072 INFO [Thread-2097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2097. 2024-11-19T04:42:58,072 DEBUG [Thread-2097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:58,072 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,072 DEBUG [Thread-2097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:58,073 INFO [Thread-2095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2095. 2024-11-19T04:42:58,073 DEBUG [Thread-2095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:58,073 DEBUG [Thread-2095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,073 DEBUG [Thread-2095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:58,079 INFO [Thread-2093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2093. 2024-11-19T04:42:58,079 DEBUG [Thread-2093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:58,079 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,080 DEBUG [Thread-2093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:58,083 INFO [Thread-2091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2091. 2024-11-19T04:42:58,083 DEBUG [Thread-2091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:58,083 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,083 DEBUG [Thread-2091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:58,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:58,216 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3092ef60b83342a39aec2586e6330e96 2024-11-19T04:42:58,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/d0e609eb8025438485ecea6d3599b388 is 50, key is test_row_0/B:col10/1731991376694/Put/seqid=0 2024-11-19T04:42:58,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742372_1548 (size=12301) 2024-11-19T04:42:58,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991438266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991438277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35746 deadline: 1731991438577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35736 deadline: 1731991438587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:58,625 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/d0e609eb8025438485ecea6d3599b388 2024-11-19T04:42:58,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/f8626b0a16fa4a898cd9dcd4a64535b0 is 50, key is test_row_0/C:col10/1731991376694/Put/seqid=0 2024-11-19T04:42:58,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742373_1549 (size=12301) 2024-11-19T04:42:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:42:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:42:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35792 deadline: 1731991438887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:42:58,888 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 728b08bae7a306a40f52d977dcb73a88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:42:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 , the old value is region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:42:58,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., hostname=08a7f35e60d4,32975,1731991233019, seqNum=2 because the exception is null or not the one we care about 2024-11-19T04:42:59,035 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/f8626b0a16fa4a898cd9dcd4a64535b0 2024-11-19T04:42:59,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3092ef60b83342a39aec2586e6330e96 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3092ef60b83342a39aec2586e6330e96 2024-11-19T04:42:59,041 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3092ef60b83342a39aec2586e6330e96, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T04:42:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/d0e609eb8025438485ecea6d3599b388 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/d0e609eb8025438485ecea6d3599b388 2024-11-19T04:42:59,044 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/d0e609eb8025438485ecea6d3599b388, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T04:42:59,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/f8626b0a16fa4a898cd9dcd4a64535b0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/f8626b0a16fa4a898cd9dcd4a64535b0 2024-11-19T04:42:59,047 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/f8626b0a16fa4a898cd9dcd4a64535b0, entries=150, sequenceid=456, filesize=12.0 K 2024-11-19T04:42:59,048 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 728b08bae7a306a40f52d977dcb73a88 in 1243ms, sequenceid=456, compaction requested=true 2024-11-19T04:42:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:42:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:42:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-19T04:42:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-19T04:42:59,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-19T04:42:59,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4680 sec 2024-11-19T04:42:59,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 2.4730 sec 2024-11-19T04:42:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:42:59,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:42:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:42:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:42:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:42:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:42:59,089 INFO [Thread-2082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2082. 2024-11-19T04:42:59,089 DEBUG [Thread-2082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:59,089 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:59,089 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:59,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:59,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3a758e38c22c4421917927ca40e36bf2 is 50, key is test_row_0/A:col10/1731991379087/Put/seqid=0 2024-11-19T04:42:59,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742374_1550 (size=12301) 2024-11-19T04:42:59,098 INFO [Thread-2086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2086. 2024-11-19T04:42:59,098 DEBUG [Thread-2086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:42:59,098 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:59,098 DEBUG [Thread-2086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:42:59,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:42:59,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3a758e38c22c4421917927ca40e36bf2 2024-11-19T04:42:59,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/96392cfafd1c4eb6bc8d73bd2124ec12 is 50, key is test_row_0/B:col10/1731991379087/Put/seqid=0 2024-11-19T04:42:59,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742375_1551 (size=12301) 2024-11-19T04:42:59,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/96392cfafd1c4eb6bc8d73bd2124ec12 2024-11-19T04:42:59,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/4d28004862af42c0b47dea011f655b4b is 50, key is test_row_0/C:col10/1731991379087/Put/seqid=0 2024-11-19T04:42:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742376_1552 (size=12301) 2024-11-19T04:43:00,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/4d28004862af42c0b47dea011f655b4b 2024-11-19T04:43:00,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/3a758e38c22c4421917927ca40e36bf2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a758e38c22c4421917927ca40e36bf2 2024-11-19T04:43:00,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a758e38c22c4421917927ca40e36bf2, entries=150, sequenceid=470, filesize=12.0 K 2024-11-19T04:43:00,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/96392cfafd1c4eb6bc8d73bd2124ec12 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/96392cfafd1c4eb6bc8d73bd2124ec12 2024-11-19T04:43:00,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/96392cfafd1c4eb6bc8d73bd2124ec12, entries=150, sequenceid=470, filesize=12.0 K 2024-11-19T04:43:00,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/4d28004862af42c0b47dea011f655b4b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/4d28004862af42c0b47dea011f655b4b 2024-11-19T04:43:00,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/4d28004862af42c0b47dea011f655b4b, entries=150, sequenceid=470, filesize=12.0 K 2024-11-19T04:43:00,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 728b08bae7a306a40f52d977dcb73a88 in 1238ms, sequenceid=470, compaction requested=true 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:00,326 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 728b08bae7a306a40f52d977dcb73a88:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:00,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:00,326 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:00,327 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:00,327 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:00,327 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/A is initiating minor compaction (all files) 2024-11-19T04:43:00,327 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/B is initiating minor compaction (all files) 2024-11-19T04:43:00,327 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/A in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:00,327 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/B in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:00,327 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/f5967b91b75744319a2cef4aa0f2381c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3092ef60b83342a39aec2586e6330e96, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a758e38c22c4421917927ca40e36bf2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=36.9 K 2024-11-19T04:43:00,327 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/774ca2d08a8d4c8cb0ad9c9d22df67ea, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/d0e609eb8025438485ecea6d3599b388, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/96392cfafd1c4eb6bc8d73bd2124ec12] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=36.9 K 2024-11-19T04:43:00,327 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5967b91b75744319a2cef4aa0f2381c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731991376515 2024-11-19T04:43:00,327 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 774ca2d08a8d4c8cb0ad9c9d22df67ea, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731991376515 2024-11-19T04:43:00,328 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3092ef60b83342a39aec2586e6330e96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991376694 2024-11-19T04:43:00,328 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e609eb8025438485ecea6d3599b388, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991376694 2024-11-19T04:43:00,328 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a758e38c22c4421917927ca40e36bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731991377929 2024-11-19T04:43:00,328 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 96392cfafd1c4eb6bc8d73bd2124ec12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731991377929 2024-11-19T04:43:00,334 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#A#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:00,334 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/ed8b533048d64ad19235254a1c819c02 is 50, key is test_row_0/A:col10/1731991379087/Put/seqid=0 2024-11-19T04:43:00,335 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:00,336 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/f7abf224a0ac4d3596889fea6fc416cb is 50, key is test_row_0/B:col10/1731991379087/Put/seqid=0 2024-11-19T04:43:00,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742377_1553 (size=13289) 2024-11-19T04:43:00,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742378_1554 (size=13289) 2024-11-19T04:43:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-19T04:43:00,737 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:00,742 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/ed8b533048d64ad19235254a1c819c02 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/ed8b533048d64ad19235254a1c819c02 2024-11-19T04:43:00,742 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/f7abf224a0ac4d3596889fea6fc416cb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/f7abf224a0ac4d3596889fea6fc416cb 2024-11-19T04:43:00,745 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/B of 728b08bae7a306a40f52d977dcb73a88 into f7abf224a0ac4d3596889fea6fc416cb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:00,745 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/A of 728b08bae7a306a40f52d977dcb73a88 into ed8b533048d64ad19235254a1c819c02(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:43:00,745 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/B, priority=13, startTime=1731991380326; duration=0sec 2024-11-19T04:43:00,745 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/A, priority=13, startTime=1731991380326; duration=0sec 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:A 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:B 2024-11-19T04:43:00,745 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T04:43:00,746 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T04:43:00,746 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 728b08bae7a306a40f52d977dcb73a88/C is initiating minor compaction (all files) 2024-11-19T04:43:00,746 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 728b08bae7a306a40f52d977dcb73a88/C in TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:00,746 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/be528bcbf1954dcda93ac90d998e8733, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/466654d27a0f4858ae5a0cdcff9673a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/f8626b0a16fa4a898cd9dcd4a64535b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/4d28004862af42c0b47dea011f655b4b] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp, totalSize=48.8 K 2024-11-19T04:43:00,746 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting be528bcbf1954dcda93ac90d998e8733, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731991374623 2024-11-19T04:43:00,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466654d27a0f4858ae5a0cdcff9673a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731991376515 2024-11-19T04:43:00,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8626b0a16fa4a898cd9dcd4a64535b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731991376694 2024-11-19T04:43:00,747 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d28004862af42c0b47dea011f655b4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731991377929 2024-11-19T04:43:00,753 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 728b08bae7a306a40f52d977dcb73a88#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:00,753 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/e164db00ca98474d8d9555910e4836b1 is 50, key is test_row_0/C:col10/1731991379087/Put/seqid=0 2024-11-19T04:43:00,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742379_1555 (size=13255) 2024-11-19T04:43:01,160 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/e164db00ca98474d8d9555910e4836b1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/e164db00ca98474d8d9555910e4836b1 2024-11-19T04:43:01,163 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 728b08bae7a306a40f52d977dcb73a88/C of 728b08bae7a306a40f52d977dcb73a88 into e164db00ca98474d8d9555910e4836b1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:01,163 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 728b08bae7a306a40f52d977dcb73a88: 2024-11-19T04:43:01,163 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., storeName=728b08bae7a306a40f52d977dcb73a88/C, priority=12, startTime=1731991380326; duration=0sec 2024-11-19T04:43:01,163 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:01,163 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 728b08bae7a306a40f52d977dcb73a88:C 2024-11-19T04:43:01,219 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T04:43:01,769 INFO [Thread-2078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2078. 2024-11-19T04:43:01,769 DEBUG [Thread-2078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:01,769 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:01,769 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:01,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:01,798 INFO [Thread-2084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2084. 2024-11-19T04:43:01,798 DEBUG [Thread-2084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:01,798 DEBUG [Thread-2084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:01,798 DEBUG [Thread-2084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:01,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:02,939 INFO [Thread-2080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2080. 2024-11-19T04:43:02,939 DEBUG [Thread-2080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:02,939 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:02,939 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:02,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3453 rows 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1149 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3447 rows 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1152 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3449 rows 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3447 rows 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1153 2024-11-19T04:43:02,939 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3454 rows 2024-11-19T04:43:02,939 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:43:02,940 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:43:02,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:02,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:02,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:02,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T04:43:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T04:43:02,943 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991382943"}]},"ts":"1731991382943"} 2024-11-19T04:43:02,944 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T04:43:02,944 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T04:43:02,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:43:02,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, UNASSIGN}] 2024-11-19T04:43:02,946 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, UNASSIGN 2024-11-19T04:43:02,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=728b08bae7a306a40f52d977dcb73a88, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:02,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, UNASSIGN because future has completed 2024-11-19T04:43:02,948 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:43:02,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:43:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T04:43:03,101 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 728b08bae7a306a40f52d977dcb73a88, disabling compactions & flushes 2024-11-19T04:43:03,101 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. after waiting 0 ms 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:03,101 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 728b08bae7a306a40f52d977dcb73a88 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=A 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=B 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 728b08bae7a306a40f52d977dcb73a88, store=C 2024-11-19T04:43:03,101 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:03,105 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/be447a2737e44b9dbedb192682814053 is 50, key is test_row_1/A:col10/1731991381767/Put/seqid=0 2024-11-19T04:43:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742380_1556 (size=9857) 2024-11-19T04:43:03,108 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/be447a2737e44b9dbedb192682814053 2024-11-19T04:43:03,114 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/7cf59848f7eb405aa6a59326794dc32c is 50, key is test_row_1/B:col10/1731991381767/Put/seqid=0 2024-11-19T04:43:03,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742381_1557 (size=9857) 2024-11-19T04:43:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T04:43:03,517 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/7cf59848f7eb405aa6a59326794dc32c 2024-11-19T04:43:03,523 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/9407fa7c7fde42aa9b7f96d1144a2d77 is 50, key is test_row_1/C:col10/1731991381767/Put/seqid=0 2024-11-19T04:43:03,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742382_1558 (size=9857) 2024-11-19T04:43:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T04:43:03,926 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/9407fa7c7fde42aa9b7f96d1144a2d77 2024-11-19T04:43:03,929 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/A/be447a2737e44b9dbedb192682814053 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/be447a2737e44b9dbedb192682814053 2024-11-19T04:43:03,932 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/be447a2737e44b9dbedb192682814053, entries=100, sequenceid=480, filesize=9.6 K 2024-11-19T04:43:03,933 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/B/7cf59848f7eb405aa6a59326794dc32c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/7cf59848f7eb405aa6a59326794dc32c 2024-11-19T04:43:03,935 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/7cf59848f7eb405aa6a59326794dc32c, entries=100, sequenceid=480, filesize=9.6 K 2024-11-19T04:43:03,936 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/.tmp/C/9407fa7c7fde42aa9b7f96d1144a2d77 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/9407fa7c7fde42aa9b7f96d1144a2d77 2024-11-19T04:43:03,939 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/9407fa7c7fde42aa9b7f96d1144a2d77, entries=100, sequenceid=480, filesize=9.6 K 2024-11-19T04:43:03,939 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 728b08bae7a306a40f52d977dcb73a88 in 838ms, sequenceid=480, compaction requested=false 2024-11-19T04:43:03,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9f558aa66d664996a882087e04d5841a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9201bf6dc4894287bb07a063fd842407, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5a351085b4a640bfac58f0d914cbbd3d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d6fb23434d614daa840a502ff88e4581, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d5156b475a1046c1865fd7e610ba230c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5fa5ce18033146e3b9edc97d8d145085, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b029dd4c3e6444fcbfce59f7a5ec9179, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/13dd39ab7c764a32995df5deead98a0a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b05be543a9ee44babae280661c56de49, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/2b526ddd63bf4d2ca0e282345659c25b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/1a5d83f495c2451bbc3d48fc7e9d5e2f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/cce06a0459304cbaad70b127c5ed2c83, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b1f8a1808ab846789c00eb7bc47bb8c9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/bb1b513843c14676864a5a8736b5e659, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d2332bf368fd4bb7a0e10040400f5dcc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/8de52865d48b4269bedda93b0a1fb8c1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/21abf104854d42eb8853006e069d35c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/797f30384e214fdba230ab4225df4449, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a70c843f6c1426fb62f0879e58b7cf2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/6dffa3a62f0e473e89d38057781b3b5a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/e7f2c29dd18c4bb59beab161a540732e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9b12f72d81c24537a540726e9bbb4dec, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/a5c84449f561418e9938b1952bd683b9, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/dd2375a4332749099415365e49d35d15, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3174463819ab4501880aaa173b0ad0ae, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/34b64b789e064369aec3263fb4612949, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/f5967b91b75744319a2cef4aa0f2381c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3092ef60b83342a39aec2586e6330e96, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a758e38c22c4421917927ca40e36bf2] to archive 2024-11-19T04:43:03,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:43:03,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9f558aa66d664996a882087e04d5841a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9f558aa66d664996a882087e04d5841a 2024-11-19T04:43:03,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9201bf6dc4894287bb07a063fd842407 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9201bf6dc4894287bb07a063fd842407 2024-11-19T04:43:03,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5a351085b4a640bfac58f0d914cbbd3d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5a351085b4a640bfac58f0d914cbbd3d 2024-11-19T04:43:03,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d6fb23434d614daa840a502ff88e4581 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d6fb23434d614daa840a502ff88e4581 2024-11-19T04:43:03,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d5156b475a1046c1865fd7e610ba230c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d5156b475a1046c1865fd7e610ba230c 2024-11-19T04:43:03,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5fa5ce18033146e3b9edc97d8d145085 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/5fa5ce18033146e3b9edc97d8d145085 2024-11-19T04:43:03,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b029dd4c3e6444fcbfce59f7a5ec9179 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b029dd4c3e6444fcbfce59f7a5ec9179 2024-11-19T04:43:03,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/13dd39ab7c764a32995df5deead98a0a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/13dd39ab7c764a32995df5deead98a0a 2024-11-19T04:43:03,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b05be543a9ee44babae280661c56de49 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b05be543a9ee44babae280661c56de49 2024-11-19T04:43:03,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/2b526ddd63bf4d2ca0e282345659c25b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/2b526ddd63bf4d2ca0e282345659c25b 2024-11-19T04:43:03,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/1a5d83f495c2451bbc3d48fc7e9d5e2f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/1a5d83f495c2451bbc3d48fc7e9d5e2f 2024-11-19T04:43:03,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/cce06a0459304cbaad70b127c5ed2c83 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/cce06a0459304cbaad70b127c5ed2c83 2024-11-19T04:43:03,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b1f8a1808ab846789c00eb7bc47bb8c9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/b1f8a1808ab846789c00eb7bc47bb8c9 2024-11-19T04:43:03,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/bb1b513843c14676864a5a8736b5e659 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/bb1b513843c14676864a5a8736b5e659 2024-11-19T04:43:03,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d2332bf368fd4bb7a0e10040400f5dcc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/d2332bf368fd4bb7a0e10040400f5dcc 2024-11-19T04:43:03,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/8de52865d48b4269bedda93b0a1fb8c1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/8de52865d48b4269bedda93b0a1fb8c1 2024-11-19T04:43:03,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/21abf104854d42eb8853006e069d35c8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/21abf104854d42eb8853006e069d35c8 2024-11-19T04:43:03,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/797f30384e214fdba230ab4225df4449 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/797f30384e214fdba230ab4225df4449 2024-11-19T04:43:03,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a70c843f6c1426fb62f0879e58b7cf2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a70c843f6c1426fb62f0879e58b7cf2 2024-11-19T04:43:03,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/6dffa3a62f0e473e89d38057781b3b5a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/6dffa3a62f0e473e89d38057781b3b5a 2024-11-19T04:43:03,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/e7f2c29dd18c4bb59beab161a540732e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/e7f2c29dd18c4bb59beab161a540732e 2024-11-19T04:43:03,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9b12f72d81c24537a540726e9bbb4dec to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/9b12f72d81c24537a540726e9bbb4dec 2024-11-19T04:43:03,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/a5c84449f561418e9938b1952bd683b9 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/a5c84449f561418e9938b1952bd683b9 2024-11-19T04:43:03,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/dd2375a4332749099415365e49d35d15 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/dd2375a4332749099415365e49d35d15 2024-11-19T04:43:03,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3174463819ab4501880aaa173b0ad0ae to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3174463819ab4501880aaa173b0ad0ae 2024-11-19T04:43:03,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/34b64b789e064369aec3263fb4612949 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/34b64b789e064369aec3263fb4612949 2024-11-19T04:43:03,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/f5967b91b75744319a2cef4aa0f2381c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/f5967b91b75744319a2cef4aa0f2381c 2024-11-19T04:43:03,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3092ef60b83342a39aec2586e6330e96 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3092ef60b83342a39aec2586e6330e96 2024-11-19T04:43:03,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a758e38c22c4421917927ca40e36bf2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/3a758e38c22c4421917927ca40e36bf2 2024-11-19T04:43:03,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/921ecabf240d4ac1bf5573f03ff263df, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b74c85f57a5d414a9400f06a2633ae87, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/bbafa11223724db08fa8d69642d6632f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e8c427edff3748fda9c99e7af646b13b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/180278dd22764adb94c165a8f18cb8a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b0e422c1d2894bc1abd7b6c0c53a4ea0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c0b64bd4c2be48fa9c97c1aa955e0278, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/ba6e15a7c7e44865943df9d4f18a30e7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/519f748bda424d3b9096de3cc67e1a8c, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2ba94178c1c24468a72c64d681d12f1d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b217bea760ac47c9bafbdd21c64eedb3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/22c06784260441a1a6f00fbf9507156d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/39ce01761f024c87af066e302c8ef749, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c172f714a13b4453b7b6b3f940c3400b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b19eaeddea5142e4a7a2c20c6ff2c311, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/833e71df945e487f83f9ee6428cf0bf2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/4d884b485dd74479aaf2ef52457ef0b8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/6ae6b76a8c94459b8d3bff89a194a7b6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/51261088ae594280a07dde27748dfb02, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/588e5b3e7eb24d918190995fa34c9d2f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/36d3459a25344e3aa62eef007ac8a7c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/0ca6b87c66104f9091c640d7b4a355c3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e613eb608f0841bf9cf78a769ca14a04, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/776f4887db6c464c903bf18dff4ba5aa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/774ca2d08a8d4c8cb0ad9c9d22df67ea, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2d86cf8f72464daf880f5f85acd0676d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/d0e609eb8025438485ecea6d3599b388, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/96392cfafd1c4eb6bc8d73bd2124ec12] to archive 2024-11-19T04:43:03,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:43:03,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/921ecabf240d4ac1bf5573f03ff263df to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/921ecabf240d4ac1bf5573f03ff263df 2024-11-19T04:43:03,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b74c85f57a5d414a9400f06a2633ae87 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b74c85f57a5d414a9400f06a2633ae87 2024-11-19T04:43:03,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/bbafa11223724db08fa8d69642d6632f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/bbafa11223724db08fa8d69642d6632f 2024-11-19T04:43:03,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e8c427edff3748fda9c99e7af646b13b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e8c427edff3748fda9c99e7af646b13b 2024-11-19T04:43:03,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/180278dd22764adb94c165a8f18cb8a3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/180278dd22764adb94c165a8f18cb8a3 2024-11-19T04:43:03,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b0e422c1d2894bc1abd7b6c0c53a4ea0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b0e422c1d2894bc1abd7b6c0c53a4ea0 2024-11-19T04:43:03,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c0b64bd4c2be48fa9c97c1aa955e0278 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c0b64bd4c2be48fa9c97c1aa955e0278 2024-11-19T04:43:03,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/ba6e15a7c7e44865943df9d4f18a30e7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/ba6e15a7c7e44865943df9d4f18a30e7 2024-11-19T04:43:03,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/519f748bda424d3b9096de3cc67e1a8c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/519f748bda424d3b9096de3cc67e1a8c 2024-11-19T04:43:03,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2ba94178c1c24468a72c64d681d12f1d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2ba94178c1c24468a72c64d681d12f1d 2024-11-19T04:43:03,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b217bea760ac47c9bafbdd21c64eedb3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b217bea760ac47c9bafbdd21c64eedb3 2024-11-19T04:43:03,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/a49ebedd4ed84c1d9d30fcdf7c91a8b3 2024-11-19T04:43:03,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/22c06784260441a1a6f00fbf9507156d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/22c06784260441a1a6f00fbf9507156d 2024-11-19T04:43:03,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/39ce01761f024c87af066e302c8ef749 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/39ce01761f024c87af066e302c8ef749 2024-11-19T04:43:03,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c172f714a13b4453b7b6b3f940c3400b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/c172f714a13b4453b7b6b3f940c3400b 2024-11-19T04:43:03,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b19eaeddea5142e4a7a2c20c6ff2c311 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/b19eaeddea5142e4a7a2c20c6ff2c311 2024-11-19T04:43:03,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/833e71df945e487f83f9ee6428cf0bf2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/833e71df945e487f83f9ee6428cf0bf2 2024-11-19T04:43:03,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/4d884b485dd74479aaf2ef52457ef0b8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/4d884b485dd74479aaf2ef52457ef0b8 2024-11-19T04:43:03,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/6ae6b76a8c94459b8d3bff89a194a7b6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/6ae6b76a8c94459b8d3bff89a194a7b6 2024-11-19T04:43:03,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/51261088ae594280a07dde27748dfb02 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/51261088ae594280a07dde27748dfb02 2024-11-19T04:43:03,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/588e5b3e7eb24d918190995fa34c9d2f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/588e5b3e7eb24d918190995fa34c9d2f 2024-11-19T04:43:03,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/36d3459a25344e3aa62eef007ac8a7c8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/36d3459a25344e3aa62eef007ac8a7c8 2024-11-19T04:43:03,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/0ca6b87c66104f9091c640d7b4a355c3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/0ca6b87c66104f9091c640d7b4a355c3 2024-11-19T04:43:03,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e613eb608f0841bf9cf78a769ca14a04 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/e613eb608f0841bf9cf78a769ca14a04 2024-11-19T04:43:03,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/776f4887db6c464c903bf18dff4ba5aa to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/776f4887db6c464c903bf18dff4ba5aa 2024-11-19T04:43:03,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/774ca2d08a8d4c8cb0ad9c9d22df67ea to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/774ca2d08a8d4c8cb0ad9c9d22df67ea 2024-11-19T04:43:03,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2d86cf8f72464daf880f5f85acd0676d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/2d86cf8f72464daf880f5f85acd0676d 2024-11-19T04:43:03,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/d0e609eb8025438485ecea6d3599b388 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/d0e609eb8025438485ecea6d3599b388 2024-11-19T04:43:03,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/96392cfafd1c4eb6bc8d73bd2124ec12 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/96392cfafd1c4eb6bc8d73bd2124ec12 2024-11-19T04:43:03,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/18da0ec8d49643e7a28725d1f601d632, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/45433f638a0d4ca48eb718ebf3a691e6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/580cf2e51c80412ab798998afd2e837a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2db574467fbf47a7a1c978a5e7cd5832, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/fc31e1fe7d9a4c519b1fc889c3ab8655, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/8b2b60cd8e0a43e0b58da08808656a98, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1e62e9c1f88c4891bd82d1d3a120f72a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/dd049ba7cb9c4c338b58493223cd41ef, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/c420beab3de04ee08df061ffd243fe22, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/d4853bfc98a1437786e7b541d7ff66c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/6b3d5927fbfe48ee99e2b4a1706c6aaa, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/de22f8d757184fdb9aab266ee75c646e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/ae81668a4bf34dd5981a5e1d3f1629b5, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/92ee0179bd504d6f9a2d7fa7c524ce3f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/65e9075de9484d3b8176f14fac0e0bd3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/02f73babe88b4a3b8682fc48a75bc62b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/83504ccab7d54db1ab47cf29d2e3d7e6, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a04c4fbc3cf743ee9cca0df826cee0ed, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/82053f4a4680452a96683e5581ace930, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/67628342eeae4e3db1cc5c21cd8ee497, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1a3205dc09c040adbd212212bc9139ff, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/7f00d83ba55e45e798f50d8503281b10, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a0c93601fd854672919b0d5f55260317, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/be528bcbf1954dcda93ac90d998e8733, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2d450fde4dc7421aba40c27cd52c8ad0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/466654d27a0f4858ae5a0cdcff9673a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/f8626b0a16fa4a898cd9dcd4a64535b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/4d28004862af42c0b47dea011f655b4b] to archive 2024-11-19T04:43:03,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:43:03,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/18da0ec8d49643e7a28725d1f601d632 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/18da0ec8d49643e7a28725d1f601d632 2024-11-19T04:43:03,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/45433f638a0d4ca48eb718ebf3a691e6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/45433f638a0d4ca48eb718ebf3a691e6 2024-11-19T04:43:03,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/580cf2e51c80412ab798998afd2e837a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/580cf2e51c80412ab798998afd2e837a 2024-11-19T04:43:03,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2db574467fbf47a7a1c978a5e7cd5832 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2db574467fbf47a7a1c978a5e7cd5832 2024-11-19T04:43:03,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/fc31e1fe7d9a4c519b1fc889c3ab8655 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/fc31e1fe7d9a4c519b1fc889c3ab8655 2024-11-19T04:43:03,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/8b2b60cd8e0a43e0b58da08808656a98 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/8b2b60cd8e0a43e0b58da08808656a98 2024-11-19T04:43:03,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1e62e9c1f88c4891bd82d1d3a120f72a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1e62e9c1f88c4891bd82d1d3a120f72a 2024-11-19T04:43:03,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/dd049ba7cb9c4c338b58493223cd41ef to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/dd049ba7cb9c4c338b58493223cd41ef 2024-11-19T04:43:03,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/c420beab3de04ee08df061ffd243fe22 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/c420beab3de04ee08df061ffd243fe22 2024-11-19T04:43:03,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/d4853bfc98a1437786e7b541d7ff66c8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/d4853bfc98a1437786e7b541d7ff66c8 2024-11-19T04:43:03,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/6b3d5927fbfe48ee99e2b4a1706c6aaa to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/6b3d5927fbfe48ee99e2b4a1706c6aaa 2024-11-19T04:43:03,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/de22f8d757184fdb9aab266ee75c646e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/de22f8d757184fdb9aab266ee75c646e 2024-11-19T04:43:03,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/ae81668a4bf34dd5981a5e1d3f1629b5 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/ae81668a4bf34dd5981a5e1d3f1629b5 2024-11-19T04:43:04,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/92ee0179bd504d6f9a2d7fa7c524ce3f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/92ee0179bd504d6f9a2d7fa7c524ce3f 2024-11-19T04:43:04,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/65e9075de9484d3b8176f14fac0e0bd3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/65e9075de9484d3b8176f14fac0e0bd3 2024-11-19T04:43:04,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/02f73babe88b4a3b8682fc48a75bc62b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/02f73babe88b4a3b8682fc48a75bc62b 2024-11-19T04:43:04,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/83504ccab7d54db1ab47cf29d2e3d7e6 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/83504ccab7d54db1ab47cf29d2e3d7e6 2024-11-19T04:43:04,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a04c4fbc3cf743ee9cca0df826cee0ed to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a04c4fbc3cf743ee9cca0df826cee0ed 2024-11-19T04:43:04,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/82053f4a4680452a96683e5581ace930 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/82053f4a4680452a96683e5581ace930 2024-11-19T04:43:04,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/67628342eeae4e3db1cc5c21cd8ee497 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/67628342eeae4e3db1cc5c21cd8ee497 2024-11-19T04:43:04,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1a3205dc09c040adbd212212bc9139ff to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/1a3205dc09c040adbd212212bc9139ff 2024-11-19T04:43:04,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/7f00d83ba55e45e798f50d8503281b10 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/7f00d83ba55e45e798f50d8503281b10 2024-11-19T04:43:04,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a0c93601fd854672919b0d5f55260317 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/a0c93601fd854672919b0d5f55260317 2024-11-19T04:43:04,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/be528bcbf1954dcda93ac90d998e8733 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/be528bcbf1954dcda93ac90d998e8733 2024-11-19T04:43:04,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2d450fde4dc7421aba40c27cd52c8ad0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/2d450fde4dc7421aba40c27cd52c8ad0 2024-11-19T04:43:04,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/466654d27a0f4858ae5a0cdcff9673a3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/466654d27a0f4858ae5a0cdcff9673a3 2024-11-19T04:43:04,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/f8626b0a16fa4a898cd9dcd4a64535b0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/f8626b0a16fa4a898cd9dcd4a64535b0 2024-11-19T04:43:04,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/4d28004862af42c0b47dea011f655b4b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/4d28004862af42c0b47dea011f655b4b 2024-11-19T04:43:04,014 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/recovered.edits/483.seqid, newMaxSeqId=483, maxSeqId=1 2024-11-19T04:43:04,014 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. 2024-11-19T04:43:04,015 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 728b08bae7a306a40f52d977dcb73a88: Waiting for close lock at 1731991383101Running coprocessor pre-close hooks at 1731991383101Disabling compacts and flushes for region at 1731991383101Disabling writes for close at 1731991383101Obtaining lock to block concurrent updates at 1731991383101Preparing flush snapshotting stores in 728b08bae7a306a40f52d977dcb73a88 at 1731991383101Finished memstore snapshotting TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731991383101Flushing stores of TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88. at 1731991383102 (+1 ms)Flushing 728b08bae7a306a40f52d977dcb73a88/A: creating writer at 1731991383102Flushing 728b08bae7a306a40f52d977dcb73a88/A: appending metadata at 1731991383104 (+2 ms)Flushing 728b08bae7a306a40f52d977dcb73a88/A: closing flushed file at 1731991383104Flushing 728b08bae7a306a40f52d977dcb73a88/B: creating writer at 1731991383111 (+7 ms)Flushing 728b08bae7a306a40f52d977dcb73a88/B: appending metadata at 1731991383113 (+2 ms)Flushing 728b08bae7a306a40f52d977dcb73a88/B: closing flushed file at 1731991383113Flushing 728b08bae7a306a40f52d977dcb73a88/C: creating writer at 1731991383520 (+407 ms)Flushing 728b08bae7a306a40f52d977dcb73a88/C: appending metadata at 1731991383522 (+2 ms)Flushing 728b08bae7a306a40f52d977dcb73a88/C: closing flushed file at 1731991383522Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19330970: reopening flushed file at 1731991383929 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2808957: reopening flushed file at 1731991383932 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1abe3c99: reopening flushed file at 1731991383935 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 728b08bae7a306a40f52d977dcb73a88 in 838ms, sequenceid=480, compaction requested=false at 1731991383939 (+4 ms)Writing region close event to WAL at 1731991384012 (+73 ms)Running coprocessor post-close hooks at 1731991384014 (+2 ms)Closed at 1731991384014 2024-11-19T04:43:04,016 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:43:04,017 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=728b08bae7a306a40f52d977dcb73a88, regionState=CLOSED 2024-11-19T04:43:04,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:43:04,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-19T04:43:04,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 728b08bae7a306a40f52d977dcb73a88, server=08a7f35e60d4,32975,1731991233019 in 1.0710 sec 2024-11-19T04:43:04,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T04:43:04,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=728b08bae7a306a40f52d977dcb73a88, UNASSIGN in 1.0740 sec 2024-11-19T04:43:04,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-19T04:43:04,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0770 sec 2024-11-19T04:43:04,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991384024"}]},"ts":"1731991384024"} 2024-11-19T04:43:04,026 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T04:43:04,026 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T04:43:04,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0860 sec 2024-11-19T04:43:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-19T04:43:04,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:04,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T04:43:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,079 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,080 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-19T04:43:04,081 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:43:04,083 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/recovered.edits] 2024-11-19T04:43:04,085 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/be447a2737e44b9dbedb192682814053 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/be447a2737e44b9dbedb192682814053 2024-11-19T04:43:04,085 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/ed8b533048d64ad19235254a1c819c02 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/A/ed8b533048d64ad19235254a1c819c02 2024-11-19T04:43:04,087 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/7cf59848f7eb405aa6a59326794dc32c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/7cf59848f7eb405aa6a59326794dc32c 2024-11-19T04:43:04,088 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/f7abf224a0ac4d3596889fea6fc416cb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/B/f7abf224a0ac4d3596889fea6fc416cb 2024-11-19T04:43:04,089 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/9407fa7c7fde42aa9b7f96d1144a2d77 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/9407fa7c7fde42aa9b7f96d1144a2d77 2024-11-19T04:43:04,090 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/e164db00ca98474d8d9555910e4836b1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/C/e164db00ca98474d8d9555910e4836b1 2024-11-19T04:43:04,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/recovered.edits/483.seqid to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88/recovered.edits/483.seqid 2024-11-19T04:43:04,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/728b08bae7a306a40f52d977dcb73a88 2024-11-19T04:43:04,092 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T04:43:04,094 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,096 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T04:43:04,098 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T04:43:04,099 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,099 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T04:43:04,099 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731991384099"}]},"ts":"9223372036854775807"} 2024-11-19T04:43:04,101 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T04:43:04,101 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 728b08bae7a306a40f52d977dcb73a88, NAME => 'TestAcidGuarantees,,1731991355876.728b08bae7a306a40f52d977dcb73a88.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T04:43:04,101 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T04:43:04,101 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731991384101"}]},"ts":"9223372036854775807"} 2024-11-19T04:43:04,103 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T04:43:04,103 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-11-19T04:43:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-19T04:43:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T04:43:04,189 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:04,196 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 248) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=362 (was 435), ProcessCount=11 (was 11), AvailableMemoryMB=11493 (was 11520) 2024-11-19T04:43:04,204 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=362, ProcessCount=11, AvailableMemoryMB=11492 2024-11-19T04:43:04,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:43:04,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:43:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:04,207 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T04:43:04,207 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:04,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-19T04:43:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T04:43:04,208 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T04:43:04,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742383_1559 (size=1061) 2024-11-19T04:43:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T04:43:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T04:43:04,614 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04 2024-11-19T04:43:04,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742384_1560 (size=53) 2024-11-19T04:43:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T04:43:05,018 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:43:05,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5eb489b71864e4cdf6076af86acc0211, disabling compactions & flushes 2024-11-19T04:43:05,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. after waiting 0 ms 2024-11-19T04:43:05,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,019 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5eb489b71864e4cdf6076af86acc0211: Waiting for close lock at 1731991385019Disabling compacts and flushes for region at 1731991385019Disabling writes for close at 1731991385019Writing region close event to WAL at 1731991385019Closed at 1731991385019 2024-11-19T04:43:05,020 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T04:43:05,020 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731991385020"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731991385020"}]},"ts":"1731991385020"} 2024-11-19T04:43:05,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T04:43:05,022 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T04:43:05,022 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991385022"}]},"ts":"1731991385022"} 2024-11-19T04:43:05,023 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T04:43:05,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, ASSIGN}] 2024-11-19T04:43:05,024 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, ASSIGN 2024-11-19T04:43:05,025 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, ASSIGN; state=OFFLINE, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=false 2024-11-19T04:43:05,175 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:05,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, ASSIGN because future has completed 2024-11-19T04:43:05,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:43:05,331 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,331 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:43:05,331 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,331 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:43:05,332 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,332 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,333 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,334 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:43:05,334 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eb489b71864e4cdf6076af86acc0211 columnFamilyName A 2024-11-19T04:43:05,334 DEBUG [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:05,334 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(327): Store=5eb489b71864e4cdf6076af86acc0211/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:43:05,334 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,335 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:43:05,335 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eb489b71864e4cdf6076af86acc0211 columnFamilyName B 2024-11-19T04:43:05,335 DEBUG [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:05,335 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(327): Store=5eb489b71864e4cdf6076af86acc0211/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:43:05,336 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,336 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:43:05,336 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eb489b71864e4cdf6076af86acc0211 columnFamilyName C 2024-11-19T04:43:05,336 DEBUG [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:05,337 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(327): Store=5eb489b71864e4cdf6076af86acc0211/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:43:05,337 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,337 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,337 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,338 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,338 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,338 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,339 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:43:05,340 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,341 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T04:43:05,342 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 5eb489b71864e4cdf6076af86acc0211; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66034679, jitterRate=-0.016006603837013245}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:43:05,342 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:05,342 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 5eb489b71864e4cdf6076af86acc0211: Running coprocessor pre-open hook at 1731991385332Writing region info on filesystem at 1731991385332Initializing all the Stores at 1731991385332Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991385332Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991385332Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991385332Cleaning up temporary data from old regions at 1731991385338 (+6 ms)Running coprocessor post-open hooks at 1731991385342 (+4 ms)Region opened successfully at 1731991385342 2024-11-19T04:43:05,343 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., pid=154, masterSystemTime=1731991385329 2024-11-19T04:43:05,344 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,344 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:05,345 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=OPEN, openSeqNum=2, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:05,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:43:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T04:43:05,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-19T04:43:05,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 in 170 msec 2024-11-19T04:43:05,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T04:43:05,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, ASSIGN in 325 msec 2024-11-19T04:43:05,350 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T04:43:05,350 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991385350"}]},"ts":"1731991385350"} 2024-11-19T04:43:05,351 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T04:43:05,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T04:43:05,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-19T04:43:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T04:43:06,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:06,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:06,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:06,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:06,360 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:06,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:06,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:06,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbec9c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:06,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:06,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:06,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:06,361 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145db6fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:06,362 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:06,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:06,364 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T04:43:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 08a7f35e60d4,46599,1731991232199 2024-11-19T04:43:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68765e5d 2024-11-19T04:43:06,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T04:43:06,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T04:43:06,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T04:43:06,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T04:43:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:06,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742385_1561 (size=1097) 2024-11-19T04:43:06,777 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-19T04:43:06,777 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-19T04:43:06,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:43:06,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, REOPEN/MOVE}] 2024-11-19T04:43:06,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, REOPEN/MOVE 2024-11-19T04:43:06,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:06,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, REOPEN/MOVE because future has completed 2024-11-19T04:43:06,783 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:43:06,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:43:06,935 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:06,935 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:43:06,935 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 5eb489b71864e4cdf6076af86acc0211, disabling compactions & flushes 2024-11-19T04:43:06,935 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:06,935 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:06,935 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. after waiting 0 ms 2024-11-19T04:43:06,935 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:06,938 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T04:43:06,939 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:06,939 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 5eb489b71864e4cdf6076af86acc0211: Waiting for close lock at 1731991386935Running coprocessor pre-close hooks at 1731991386935Disabling compacts and flushes for region at 1731991386935Disabling writes for close at 1731991386935Writing region close event to WAL at 1731991386936 (+1 ms)Running coprocessor post-close hooks at 1731991386939 (+3 ms)Closed at 1731991386939 2024-11-19T04:43:06,939 WARN [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 5eb489b71864e4cdf6076af86acc0211 to self. 2024-11-19T04:43:06,940 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:06,941 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=CLOSED 2024-11-19T04:43:06,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:43:06,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-19T04:43:06,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 in 160 msec 2024-11-19T04:43:06,944 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, REOPEN/MOVE; state=CLOSED, location=08a7f35e60d4,32975,1731991233019; forceNewPlan=false, retain=true 2024-11-19T04:43:07,095 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=OPENING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, REOPEN/MOVE because future has completed 2024-11-19T04:43:07,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:43:07,251 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,251 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} 2024-11-19T04:43:07,251 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,251 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T04:43:07,251 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,251 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,252 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,253 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:43:07,253 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eb489b71864e4cdf6076af86acc0211 columnFamilyName A 2024-11-19T04:43:07,254 DEBUG [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:07,254 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(327): Store=5eb489b71864e4cdf6076af86acc0211/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:43:07,255 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,255 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:43:07,255 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eb489b71864e4cdf6076af86acc0211 columnFamilyName B 2024-11-19T04:43:07,255 DEBUG [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:07,255 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(327): Store=5eb489b71864e4cdf6076af86acc0211/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:43:07,255 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,256 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-19T04:43:07,256 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eb489b71864e4cdf6076af86acc0211 columnFamilyName C 2024-11-19T04:43:07,256 DEBUG [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:07,256 INFO [StoreOpener-5eb489b71864e4cdf6076af86acc0211-1 {}] regionserver.HStore(327): Store=5eb489b71864e4cdf6076af86acc0211/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T04:43:07,256 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,256 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,257 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,258 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,258 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,258 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,259 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T04:43:07,259 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,260 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 5eb489b71864e4cdf6076af86acc0211; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72623530, jitterRate=0.08217492699623108}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T04:43:07,260 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,261 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 5eb489b71864e4cdf6076af86acc0211: Running coprocessor pre-open hook at 1731991387251Writing region info on filesystem at 1731991387251Initializing all the Stores at 1731991387252 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991387252Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991387252Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731991387252Cleaning up temporary data from old regions at 1731991387258 (+6 ms)Running coprocessor post-open hooks at 1731991387260 (+2 ms)Region opened successfully at 1731991387261 (+1 ms) 2024-11-19T04:43:07,261 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., pid=159, masterSystemTime=1731991387248 2024-11-19T04:43:07,263 DEBUG [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,263 INFO [RS_OPEN_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,263 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=OPEN, openSeqNum=5, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:43:07,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-19T04:43:07,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 in 169 msec 2024-11-19T04:43:07,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T04:43:07,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, REOPEN/MOVE in 485 msec 2024-11-19T04:43:07,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-19T04:43:07,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-19T04:43:07,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-19T04:43:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-19T04:43:07,272 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:07,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e42277c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4a69db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,276 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8015c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,278 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2aa9e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,279 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae18322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e28cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d25ea65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28aebd91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,284 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@83938f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf2ca25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,287 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d7895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23207e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,289 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4babc00b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,290 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d62e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@449fc76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa1351f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78eb3440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12751c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 08a7f35e60d4,46599,-1 for getting cluster id 2024-11-19T04:43:07,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T04:43:07,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f524422f-86ed-4a7c-aeb0-e6f8c33d39a3' 2024-11-19T04:43:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T04:43:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f524422f-86ed-4a7c-aeb0-e6f8c33d39a3" 2024-11-19T04:43:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2b6334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [08a7f35e60d4,46599,-1] 2024-11-19T04:43:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T04:43:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:07,298 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T04:43:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f661e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T04:43:07,299 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,299 DEBUG [Thread-2503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,299 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:43:07,299 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,300 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,300 DEBUG [Thread-2501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,301 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:43:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:07,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,304 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:43:07,304 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:43:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,305 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,306 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,307 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T04:43:07,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=08a7f35e60d4,32975,1731991233019, seqNum=-1] 2024-11-19T04:43:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T04:43:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T04:43:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5] 2024-11-19T04:43:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-19T04:43:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:07,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991387327, max timestamp=1731991387334Num uniques 859; 2024-11-19T04:43:07,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:07,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991387327, max timestamp=1731991387334Num uniques 859; 2024-11-19T04:43:07,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:07,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991387327, max timestamp=1731991387334Num uniques 859; 2024-11-19T04:43:07,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:07,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bd85ad16578546b4b69a03cef4b6efa1_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991387314/Put/seqid=0 2024-11-19T04:43:07,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742386_1562 (size=9714) 2024-11-19T04:43:07,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991447352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991447352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991447353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991447358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991447358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:07,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:07,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:07,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991447467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991447467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991447468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991447468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991447468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991447676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991447677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991447677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991447677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991447677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:07,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119bd85ad16578546b4b69a03cef4b6efa1_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bd85ad16578546b4b69a03cef4b6efa1_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:07,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/a681d0ca182642afa4b1aa8d31cdee85, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:07,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/a681d0ca182642afa4b1aa8d31cdee85 is 175, key is test_row_0/A:col10/1731991387314/Put/seqid=0 2024-11-19T04:43:07,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742387_1563 (size=22361) 2024-11-19T04:43:07,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:07,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991447986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991447986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991447987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991447987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991447987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:08,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/a681d0ca182642afa4b1aa8d31cdee85 2024-11-19T04:43:08,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/1046fa8bfd214bda95c06b90287db559 is 50, key is test_row_0/B:col10/1731991387314/Put/seqid=0 2024-11-19T04:43:08,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742388_1564 (size=9657) 2024-11-19T04:43:08,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/1046fa8bfd214bda95c06b90287db559 2024-11-19T04:43:08,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/76fe7279604d45ff8ef9b0d771764e8b is 50, key is test_row_0/C:col10/1731991387314/Put/seqid=0 2024-11-19T04:43:08,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742389_1565 (size=9657) 2024-11-19T04:43:08,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:08,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991448497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991448497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991448497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991448497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991448497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:08,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:08,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/76fe7279604d45ff8ef9b0d771764e8b 2024-11-19T04:43:08,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/a681d0ca182642afa4b1aa8d31cdee85 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85 2024-11-19T04:43:08,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85, entries=100, sequenceid=15, filesize=21.8 K 2024-11-19T04:43:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/1046fa8bfd214bda95c06b90287db559 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/1046fa8bfd214bda95c06b90287db559 2024-11-19T04:43:08,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/1046fa8bfd214bda95c06b90287db559, entries=100, sequenceid=15, filesize=9.4 K 2024-11-19T04:43:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/76fe7279604d45ff8ef9b0d771764e8b as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/76fe7279604d45ff8ef9b0d771764e8b 2024-11-19T04:43:08,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/76fe7279604d45ff8ef9b0d771764e8b, entries=100, sequenceid=15, filesize=9.4 K 2024-11-19T04:43:08,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 5eb489b71864e4cdf6076af86acc0211 in 1320ms, sequenceid=15, compaction requested=false 2024-11-19T04:43:08,634 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T04:43:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:08,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-19T04:43:08,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:08,680 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-19T04:43:08,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:08,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:08,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:08,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:08,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:08,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dfd3f051e25b4722b71380e1e09b0540_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991387357/Put/seqid=0 2024-11-19T04:43:08,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742390_1566 (size=12154) 2024-11-19T04:43:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:09,095 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dfd3f051e25b4722b71380e1e09b0540_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dfd3f051e25b4722b71380e1e09b0540_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/7888e7ac98294ac5a5ea928e6510e1d7, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/7888e7ac98294ac5a5ea928e6510e1d7 is 175, key is test_row_0/A:col10/1731991387357/Put/seqid=0 2024-11-19T04:43:09,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742391_1567 (size=30955) 2024-11-19T04:43:09,100 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/7888e7ac98294ac5a5ea928e6510e1d7 2024-11-19T04:43:09,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/010776f760e94a3c8efc2f14bc4210c1 is 50, key is test_row_0/B:col10/1731991387357/Put/seqid=0 2024-11-19T04:43:09,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742392_1568 (size=12001) 2024-11-19T04:43:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:09,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:09,510 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/010776f760e94a3c8efc2f14bc4210c1 2024-11-19T04:43:09,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/ace198dd688a4d1ca2e8b2f493fdc7ef is 50, key is test_row_0/C:col10/1731991387357/Put/seqid=0 2024-11-19T04:43:09,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742393_1569 (size=12001) 2024-11-19T04:43:09,522 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/ace198dd688a4d1ca2e8b2f493fdc7ef 2024-11-19T04:43:09,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991449521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991449522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/7888e7ac98294ac5a5ea928e6510e1d7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7 2024-11-19T04:43:09,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991449522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991449523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991449524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,530 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7, entries=150, sequenceid=51, filesize=30.2 K 2024-11-19T04:43:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/010776f760e94a3c8efc2f14bc4210c1 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/010776f760e94a3c8efc2f14bc4210c1 2024-11-19T04:43:09,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T04:43:09,536 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/010776f760e94a3c8efc2f14bc4210c1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T04:43:09,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/ace198dd688a4d1ca2e8b2f493fdc7ef as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ace198dd688a4d1ca2e8b2f493fdc7ef 2024-11-19T04:43:09,541 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ace198dd688a4d1ca2e8b2f493fdc7ef, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T04:43:09,542 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 5eb489b71864e4cdf6076af86acc0211 in 862ms, sequenceid=51, compaction requested=false 2024-11-19T04:43:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-19T04:43:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-19T04:43:09,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T04:43:09,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-19T04:43:09,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2490 sec 2024-11-19T04:43:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:09,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:43:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:09,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119471095bec4544400987c70182555361c_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:09,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742394_1570 (size=17034) 2024-11-19T04:43:09,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:09,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119471095bec4544400987c70182555361c_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119471095bec4544400987c70182555361c_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:09,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/c6570403225a456a8e2fc9f7e71b96c2, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:09,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/c6570403225a456a8e2fc9f7e71b96c2 is 175, key is test_row_0/A:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:09,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742395_1571 (size=48139) 2024-11-19T04:43:09,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991449661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991449661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991449664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991449664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991449666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991449777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991449777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991449777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991449777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991449777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991449987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991449987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991449987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991449987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991449987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:09,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=67, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/c6570403225a456a8e2fc9f7e71b96c2 2024-11-19T04:43:10,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/335b17b47c6041eab862b7f83c8c69c8 is 50, key is test_row_0/B:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:10,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742396_1572 (size=12001) 2024-11-19T04:43:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991450297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991450297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991450297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991450297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991450297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/335b17b47c6041eab862b7f83c8c69c8 2024-11-19T04:43:10,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/61f2ac82701d4a019de6c83672fdb6b3 is 50, key is test_row_0/C:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:10,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742397_1573 (size=12001) 2024-11-19T04:43:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991450807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991450807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991450807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991450807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991450807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:10,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/61f2ac82701d4a019de6c83672fdb6b3 2024-11-19T04:43:10,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/c6570403225a456a8e2fc9f7e71b96c2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2 2024-11-19T04:43:10,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2, entries=250, sequenceid=67, filesize=47.0 K 2024-11-19T04:43:10,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/335b17b47c6041eab862b7f83c8c69c8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/335b17b47c6041eab862b7f83c8c69c8 2024-11-19T04:43:10,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/335b17b47c6041eab862b7f83c8c69c8, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T04:43:10,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/61f2ac82701d4a019de6c83672fdb6b3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f2ac82701d4a019de6c83672fdb6b3 2024-11-19T04:43:10,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f2ac82701d4a019de6c83672fdb6b3, entries=150, sequenceid=67, filesize=11.7 K 2024-11-19T04:43:10,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5eb489b71864e4cdf6076af86acc0211 in 1261ms, sequenceid=67, compaction requested=true 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:10,900 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:10,900 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:10,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:10,901 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:10,901 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/A is initiating minor compaction (all files) 2024-11-19T04:43:10,901 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/A in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:10,901 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=99.1 K 2024-11-19T04:43:10,901 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:10,901 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2] 2024-11-19T04:43:10,902 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:10,902 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/B is initiating minor compaction (all files) 2024-11-19T04:43:10,902 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/B in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:10,902 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/1046fa8bfd214bda95c06b90287db559, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/010776f760e94a3c8efc2f14bc4210c1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/335b17b47c6041eab862b7f83c8c69c8] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=32.9 K 2024-11-19T04:43:10,902 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting a681d0ca182642afa4b1aa8d31cdee85, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991387312 2024-11-19T04:43:10,902 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1046fa8bfd214bda95c06b90287db559, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991387312 2024-11-19T04:43:10,902 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7888e7ac98294ac5a5ea928e6510e1d7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731991387352 2024-11-19T04:43:10,903 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6570403225a456a8e2fc9f7e71b96c2, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731991389509 2024-11-19T04:43:10,903 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 010776f760e94a3c8efc2f14bc4210c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731991387352 2024-11-19T04:43:10,903 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 335b17b47c6041eab862b7f83c8c69c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731991389638 2024-11-19T04:43:10,909 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:10,912 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:10,913 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/6751e0e303d642df912d10546a392dbd is 50, key is test_row_0/B:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:10,913 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195c9dd195d7e540f5bdfbaa8171cc2645_5eb489b71864e4cdf6076af86acc0211 store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:10,915 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195c9dd195d7e540f5bdfbaa8171cc2645_5eb489b71864e4cdf6076af86acc0211, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:10,916 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195c9dd195d7e540f5bdfbaa8171cc2645_5eb489b71864e4cdf6076af86acc0211 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:10,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742398_1574 (size=12104) 2024-11-19T04:43:10,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742399_1575 (size=4469) 2024-11-19T04:43:10,939 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#A#compaction#488 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:10,940 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/13c2fa61236b4d2bb5b6cd3b471fb4a3 is 175, key is test_row_0/A:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:10,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742400_1576 (size=31058) 2024-11-19T04:43:10,950 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/13c2fa61236b4d2bb5b6cd3b471fb4a3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/13c2fa61236b4d2bb5b6cd3b471fb4a3 2024-11-19T04:43:10,956 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/A of 5eb489b71864e4cdf6076af86acc0211 into 13c2fa61236b4d2bb5b6cd3b471fb4a3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:10,956 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:10,957 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/A, priority=13, startTime=1731991390900; duration=0sec 2024-11-19T04:43:10,957 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:10,957 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:A 2024-11-19T04:43:10,957 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:10,958 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:10,958 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/C is initiating minor compaction (all files) 2024-11-19T04:43:10,958 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/C in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:10,958 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/76fe7279604d45ff8ef9b0d771764e8b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ace198dd688a4d1ca2e8b2f493fdc7ef, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f2ac82701d4a019de6c83672fdb6b3] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=32.9 K 2024-11-19T04:43:10,959 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76fe7279604d45ff8ef9b0d771764e8b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731991387312 2024-11-19T04:43:10,959 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ace198dd688a4d1ca2e8b2f493fdc7ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731991387352 2024-11-19T04:43:10,960 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f2ac82701d4a019de6c83672fdb6b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731991389638 2024-11-19T04:43:10,968 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:10,969 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/543641bb6f234715bbe5c70a5a1381b0 is 50, key is test_row_0/C:col10/1731991389638/Put/seqid=0 2024-11-19T04:43:10,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742401_1577 (size=12104) 2024-11-19T04:43:11,327 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/6751e0e303d642df912d10546a392dbd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6751e0e303d642df912d10546a392dbd 2024-11-19T04:43:11,331 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/B of 5eb489b71864e4cdf6076af86acc0211 into 6751e0e303d642df912d10546a392dbd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:11,331 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:11,331 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/B, priority=13, startTime=1731991390900; duration=0sec 2024-11-19T04:43:11,331 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:11,331 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:B 2024-11-19T04:43:11,382 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/543641bb6f234715bbe5c70a5a1381b0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/543641bb6f234715bbe5c70a5a1381b0 2024-11-19T04:43:11,386 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/C of 5eb489b71864e4cdf6076af86acc0211 into 543641bb6f234715bbe5c70a5a1381b0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:11,386 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:11,386 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/C, priority=13, startTime=1731991390900; duration=0sec 2024-11-19T04:43:11,386 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:11,386 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:C 2024-11-19T04:43:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-19T04:43:11,458 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:11,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:43:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-19T04:43:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T04:43:11,460 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:43:11,461 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:43:11,461 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:43:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T04:43:11,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-19T04:43:11,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:11,614 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T04:43:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:11,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111957937056ae18445e88ebb2e863ab5e29_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991389663/Put/seqid=0 2024-11-19T04:43:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742402_1578 (size=12154) 2024-11-19T04:43:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:11,630 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111957937056ae18445e88ebb2e863ab5e29_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111957937056ae18445e88ebb2e863ab5e29_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:11,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ff50d19878c452e86313c9a48fdd9c8, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:11,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ff50d19878c452e86313c9a48fdd9c8 is 175, key is test_row_0/A:col10/1731991389663/Put/seqid=0 2024-11-19T04:43:11,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742403_1579 (size=30955) 2024-11-19T04:43:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T04:43:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:11,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:11,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991451830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991451832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991451833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991451834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991451834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991451937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991451947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991451947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991451947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991451947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ff50d19878c452e86313c9a48fdd9c8 2024-11-19T04:43:12,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/8938b5469d40465780a4f4c01ad120ec is 50, key is test_row_0/B:col10/1731991389663/Put/seqid=0 2024-11-19T04:43:12,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742404_1580 (size=12001) 2024-11-19T04:43:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T04:43:12,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991452147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991452157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991452157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991452157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991452157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,446 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/8938b5469d40465780a4f4c01ad120ec 2024-11-19T04:43:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/afecce6685b2435fb8c0fae0dc1cf987 is 50, key is test_row_0/C:col10/1731991389663/Put/seqid=0 2024-11-19T04:43:12,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742405_1581 (size=12001) 2024-11-19T04:43:12,457 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/afecce6685b2435fb8c0fae0dc1cf987 2024-11-19T04:43:12,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991452457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ff50d19878c452e86313c9a48fdd9c8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8 2024-11-19T04:43:12,464 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8, entries=150, sequenceid=86, filesize=30.2 K 2024-11-19T04:43:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/8938b5469d40465780a4f4c01ad120ec as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8938b5469d40465780a4f4c01ad120ec 2024-11-19T04:43:12,468 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8938b5469d40465780a4f4c01ad120ec, entries=150, sequenceid=86, filesize=11.7 K 2024-11-19T04:43:12,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/afecce6685b2435fb8c0fae0dc1cf987 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/afecce6685b2435fb8c0fae0dc1cf987 2024-11-19T04:43:12,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991452467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991452467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,471 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/afecce6685b2435fb8c0fae0dc1cf987, entries=150, sequenceid=86, filesize=11.7 K 2024-11-19T04:43:12,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991452467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,472 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5eb489b71864e4cdf6076af86acc0211 in 859ms, sequenceid=86, compaction requested=false 2024-11-19T04:43:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-19T04:43:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-19T04:43:12,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-19T04:43:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:12,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:12,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-19T04:43:12,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-19T04:43:12,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0190 sec 2024-11-19T04:43:12,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111939b392ff3ca04d1e85607de574a7ce6f_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:12,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742406_1582 (size=14594) 2024-11-19T04:43:12,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:12,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111939b392ff3ca04d1e85607de574a7ce6f_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939b392ff3ca04d1e85607de574a7ce6f_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:12,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/375ef09b428a42409181631cdfbabce7, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:12,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/375ef09b428a42409181631cdfbabce7 is 175, key is test_row_0/A:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742407_1583 (size=39549) 2024-11-19T04:43:12,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991452533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T04:43:12,598 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:12,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:43:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-19T04:43:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:12,600 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:43:12,601 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:43:12,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:43:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991452647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:12,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:12,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991452857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:12,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:12,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:12,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:12,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:12,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/375ef09b428a42409181631cdfbabce7 2024-11-19T04:43:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:12,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/99a2913f8cae40e0836067de2e4eb867 is 50, key is test_row_0/B:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:12,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742408_1584 (size=12001) 2024-11-19T04:43:12,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/99a2913f8cae40e0836067de2e4eb867 2024-11-19T04:43:12,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/ba3c2c9521cd49e797200b4f2f859de7 is 50, key is test_row_0/C:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742409_1585 (size=12001) 2024-11-19T04:43:12,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/ba3c2c9521cd49e797200b4f2f859de7 2024-11-19T04:43:12,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/375ef09b428a42409181631cdfbabce7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7 2024-11-19T04:43:12,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7, entries=200, sequenceid=103, filesize=38.6 K 2024-11-19T04:43:12,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/99a2913f8cae40e0836067de2e4eb867 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/99a2913f8cae40e0836067de2e4eb867 2024-11-19T04:43:12,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/99a2913f8cae40e0836067de2e4eb867, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T04:43:12,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/ba3c2c9521cd49e797200b4f2f859de7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ba3c2c9521cd49e797200b4f2f859de7 2024-11-19T04:43:12,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ba3c2c9521cd49e797200b4f2f859de7, entries=150, sequenceid=103, filesize=11.7 K 2024-11-19T04:43:12,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5eb489b71864e4cdf6076af86acc0211 in 477ms, sequenceid=103, compaction requested=true 2024-11-19T04:43:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:12,950 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:12,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:12,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:12,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:12,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:12,951 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:12,956 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:12,956 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:12,956 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/A is initiating minor compaction (all files) 2024-11-19T04:43:12,956 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/B is initiating minor compaction (all files) 2024-11-19T04:43:12,957 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/A in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,957 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/B in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,957 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6751e0e303d642df912d10546a392dbd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8938b5469d40465780a4f4c01ad120ec, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/99a2913f8cae40e0836067de2e4eb867] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=35.3 K 2024-11-19T04:43:12,957 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/13c2fa61236b4d2bb5b6cd3b471fb4a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=99.2 K 2024-11-19T04:43:12,957 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,957 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/13c2fa61236b4d2bb5b6cd3b471fb4a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7] 2024-11-19T04:43:12,957 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6751e0e303d642df912d10546a392dbd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731991389638 2024-11-19T04:43:12,957 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13c2fa61236b4d2bb5b6cd3b471fb4a3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731991389638 2024-11-19T04:43:12,957 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8938b5469d40465780a4f4c01ad120ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731991389655 2024-11-19T04:43:12,957 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff50d19878c452e86313c9a48fdd9c8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731991389655 2024-11-19T04:43:12,957 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 99a2913f8cae40e0836067de2e4eb867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731991391828 2024-11-19T04:43:12,958 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 375ef09b428a42409181631cdfbabce7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731991391828 2024-11-19T04:43:12,964 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:12,965 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:12,966 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/6ddaf75eb10f4638a0bbde9bd8df24ac is 50, key is test_row_0/B:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:12,966 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111958e6f31d0b824cb1b1c415cb943bfb02_5eb489b71864e4cdf6076af86acc0211 store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:12,968 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111958e6f31d0b824cb1b1c415cb943bfb02_5eb489b71864e4cdf6076af86acc0211, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:12,968 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111958e6f31d0b824cb1b1c415cb943bfb02_5eb489b71864e4cdf6076af86acc0211 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:12,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-19T04:43:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:12,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:12,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742410_1586 (size=12207) 2024-11-19T04:43:12,989 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/6ddaf75eb10f4638a0bbde9bd8df24ac as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6ddaf75eb10f4638a0bbde9bd8df24ac 2024-11-19T04:43:12,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742411_1587 (size=4469) 2024-11-19T04:43:12,994 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/B of 5eb489b71864e4cdf6076af86acc0211 into 6ddaf75eb10f4638a0bbde9bd8df24ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:12,994 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:12,994 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/B, priority=13, startTime=1731991392950; duration=0sec 2024-11-19T04:43:12,994 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:12,994 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:B 2024-11-19T04:43:12,994 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:12,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119de88f1cfdf554243b40b651e8bdadb9b_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991392531/Put/seqid=0 2024-11-19T04:43:12,997 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:12,997 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/C is initiating minor compaction (all files) 2024-11-19T04:43:12,997 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/C in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:12,997 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/543641bb6f234715bbe5c70a5a1381b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/afecce6685b2435fb8c0fae0dc1cf987, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ba3c2c9521cd49e797200b4f2f859de7] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=35.3 K 2024-11-19T04:43:12,998 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 543641bb6f234715bbe5c70a5a1381b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731991389638 2024-11-19T04:43:12,998 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting afecce6685b2435fb8c0fae0dc1cf987, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731991389655 2024-11-19T04:43:12,998 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3c2c9521cd49e797200b4f2f859de7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731991391828 2024-11-19T04:43:13,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742412_1588 (size=14594) 2024-11-19T04:43:13,004 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#C#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:13,004 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/db6ca58b7e5f4375aaf16cfbc93b9a83 is 50, key is test_row_0/C:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:13,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991452998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991452999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991453006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991453006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742413_1589 (size=12207) 2024-11-19T04:43:13,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991453117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991453117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991453117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991453118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991453167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991453327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991453327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991453327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991453327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,391 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:13,392 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/4d284ad67f1e40e8b89ddb46d3700145 is 175, key is test_row_0/A:col10/1731991392472/Put/seqid=0 2024-11-19T04:43:13,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742414_1590 (size=31161) 2024-11-19T04:43:13,401 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:13,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119de88f1cfdf554243b40b651e8bdadb9b_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119de88f1cfdf554243b40b651e8bdadb9b_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:13,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/d985eaa02dd344278efabeb036a70bba, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:13,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/d985eaa02dd344278efabeb036a70bba is 175, key is test_row_0/A:col10/1731991392531/Put/seqid=0 2024-11-19T04:43:13,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742415_1591 (size=39549) 2024-11-19T04:43:13,421 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/db6ca58b7e5f4375aaf16cfbc93b9a83 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/db6ca58b7e5f4375aaf16cfbc93b9a83 2024-11-19T04:43:13,425 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/C of 5eb489b71864e4cdf6076af86acc0211 into db6ca58b7e5f4375aaf16cfbc93b9a83(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:13,425 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:13,425 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/C, priority=13, startTime=1731991392951; duration=0sec 2024-11-19T04:43:13,425 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:13,425 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:C 2024-11-19T04:43:13,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991453637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991453638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991453638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991453638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991453677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:13,800 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/4d284ad67f1e40e8b89ddb46d3700145 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/4d284ad67f1e40e8b89ddb46d3700145 2024-11-19T04:43:13,803 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/A of 5eb489b71864e4cdf6076af86acc0211 into 4d284ad67f1e40e8b89ddb46d3700145(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:13,803 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:13,803 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/A, priority=13, startTime=1731991392950; duration=0sec 2024-11-19T04:43:13,804 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:13,804 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:A 2024-11-19T04:43:13,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/d985eaa02dd344278efabeb036a70bba 2024-11-19T04:43:13,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/b9e104fd60bd453ab53d71ea5a57d2d7 is 50, key is test_row_0/B:col10/1731991392531/Put/seqid=0 2024-11-19T04:43:13,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742416_1592 (size=12001) 2024-11-19T04:43:13,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:13,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:14,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991454146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991454147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991454147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:14,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991454157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:14,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/b9e104fd60bd453ab53d71ea5a57d2d7 2024-11-19T04:43:14,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/40387bbb0bfb44d1aa9aee12debe9f97 is 50, key is test_row_0/C:col10/1731991392531/Put/seqid=0 2024-11-19T04:43:14,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742417_1593 (size=12001) 2024-11-19T04:43:14,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:14,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:14,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/40387bbb0bfb44d1aa9aee12debe9f97 2024-11-19T04:43:14,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/d985eaa02dd344278efabeb036a70bba as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba 2024-11-19T04:43:14,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba, entries=200, sequenceid=119, filesize=38.6 K 2024-11-19T04:43:14,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/b9e104fd60bd453ab53d71ea5a57d2d7 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/b9e104fd60bd453ab53d71ea5a57d2d7 2024-11-19T04:43:14,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/b9e104fd60bd453ab53d71ea5a57d2d7, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T04:43:14,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/40387bbb0bfb44d1aa9aee12debe9f97 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/40387bbb0bfb44d1aa9aee12debe9f97 2024-11-19T04:43:14,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/40387bbb0bfb44d1aa9aee12debe9f97, entries=150, sequenceid=119, filesize=11.7 K 2024-11-19T04:43:14,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 5eb489b71864e4cdf6076af86acc0211 in 1693ms, sequenceid=119, compaction requested=false 2024-11-19T04:43:14,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:14,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T04:43:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:14,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:14,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dad785fb1c364eb39884bbcc9b623c7d_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:14,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742418_1594 (size=14744) 2024-11-19T04:43:14,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:14,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:14,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:14,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991454746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:14,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:14,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991454857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:14,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:14,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991455067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,110 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:15,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dad785fb1c364eb39884bbcc9b623c7d_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dad785fb1c364eb39884bbcc9b623c7d_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:15,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1d3a278e3d204f8a8e5f61487f0015c8, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:15,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1d3a278e3d204f8a8e5f61487f0015c8 is 175, key is test_row_0/A:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:15,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742419_1595 (size=39699) 2024-11-19T04:43:15,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991455156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,160 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991455167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991455167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,170 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991455167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,170 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,171 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991455376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1d3a278e3d204f8a8e5f61487f0015c8 2024-11-19T04:43:15,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/80e782629fad4081b1bc84b0f60c0841 is 50, key is test_row_0/B:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:15,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742420_1596 (size=12151) 2024-11-19T04:43:15,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991455887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:15,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/80e782629fad4081b1bc84b0f60c0841 2024-11-19T04:43:15,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/c26d05e8b7594305a08eb4f1cb2e235a is 50, key is test_row_0/C:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:15,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742421_1597 (size=12151) 2024-11-19T04:43:15,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:15,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:16,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/c26d05e8b7594305a08eb4f1cb2e235a 2024-11-19T04:43:16,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1d3a278e3d204f8a8e5f61487f0015c8 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8 2024-11-19T04:43:16,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8, entries=200, sequenceid=140, filesize=38.8 K 2024-11-19T04:43:16,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/80e782629fad4081b1bc84b0f60c0841 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/80e782629fad4081b1bc84b0f60c0841 2024-11-19T04:43:16,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/80e782629fad4081b1bc84b0f60c0841, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T04:43:16,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/c26d05e8b7594305a08eb4f1cb2e235a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/c26d05e8b7594305a08eb4f1cb2e235a 2024-11-19T04:43:16,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/c26d05e8b7594305a08eb4f1cb2e235a, entries=150, sequenceid=140, filesize=11.9 K 2024-11-19T04:43:16,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5eb489b71864e4cdf6076af86acc0211 in 1657ms, sequenceid=140, compaction requested=true 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:16,356 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:16,356 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:16,357 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:16,357 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:16,357 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/A is initiating minor compaction (all files) 2024-11-19T04:43:16,357 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/B is initiating minor compaction (all files) 2024-11-19T04:43:16,357 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/A in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,357 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/B in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,357 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/4d284ad67f1e40e8b89ddb46d3700145, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=107.8 K 2024-11-19T04:43:16,358 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6ddaf75eb10f4638a0bbde9bd8df24ac, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/b9e104fd60bd453ab53d71ea5a57d2d7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/80e782629fad4081b1bc84b0f60c0841] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=35.5 K 2024-11-19T04:43:16,358 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/4d284ad67f1e40e8b89ddb46d3700145, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8] 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ddaf75eb10f4638a0bbde9bd8df24ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731991391828 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d284ad67f1e40e8b89ddb46d3700145, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731991391828 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting d985eaa02dd344278efabeb036a70bba, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731991392513 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e104fd60bd453ab53d71ea5a57d2d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731991392525 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3a278e3d204f8a8e5f61487f0015c8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731991392987 2024-11-19T04:43:16,358 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e782629fad4081b1bc84b0f60c0841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731991393003 2024-11-19T04:43:16,364 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:16,365 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:16,365 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119cec280a3c113491caaa124ade26336f6_5eb489b71864e4cdf6076af86acc0211 store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:16,366 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/3792971d426f43a48ae680408a82a411 is 50, key is test_row_0/B:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:16,367 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119cec280a3c113491caaa124ade26336f6_5eb489b71864e4cdf6076af86acc0211, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:16,367 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cec280a3c113491caaa124ade26336f6_5eb489b71864e4cdf6076af86acc0211 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:16,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742422_1598 (size=12459) 2024-11-19T04:43:16,374 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/3792971d426f43a48ae680408a82a411 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3792971d426f43a48ae680408a82a411 2024-11-19T04:43:16,379 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/B of 5eb489b71864e4cdf6076af86acc0211 into 3792971d426f43a48ae680408a82a411(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:16,379 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:16,379 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/B, priority=13, startTime=1731991396356; duration=0sec 2024-11-19T04:43:16,379 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:16,379 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:B 2024-11-19T04:43:16,379 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:16,380 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:16,380 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/C is initiating minor compaction (all files) 2024-11-19T04:43:16,380 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/C in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,380 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/db6ca58b7e5f4375aaf16cfbc93b9a83, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/40387bbb0bfb44d1aa9aee12debe9f97, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/c26d05e8b7594305a08eb4f1cb2e235a] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=35.5 K 2024-11-19T04:43:16,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742423_1599 (size=4469) 2024-11-19T04:43:16,381 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting db6ca58b7e5f4375aaf16cfbc93b9a83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731991391828 2024-11-19T04:43:16,382 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 40387bbb0bfb44d1aa9aee12debe9f97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731991392525 2024-11-19T04:43:16,382 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting c26d05e8b7594305a08eb4f1cb2e235a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731991393003 2024-11-19T04:43:16,388 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#C#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:16,388 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/7be41dd437124c05a5e7223031947140 is 50, key is test_row_0/C:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:16,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742424_1600 (size=12459) 2024-11-19T04:43:16,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-19T04:43:16,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:16,426 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:43:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:16,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111943276dfe309e494aa5bd36553c513a13_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991394734/Put/seqid=0 2024-11-19T04:43:16,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742425_1601 (size=12304) 2024-11-19T04:43:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:16,781 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:16,782 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1aa7847de13b4971ac03779aefac3b34 is 175, key is test_row_0/A:col10/1731991393003/Put/seqid=0 2024-11-19T04:43:16,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742426_1602 (size=31413) 2024-11-19T04:43:16,804 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/7be41dd437124c05a5e7223031947140 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/7be41dd437124c05a5e7223031947140 2024-11-19T04:43:16,813 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/C of 5eb489b71864e4cdf6076af86acc0211 into 7be41dd437124c05a5e7223031947140(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:16,813 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:16,813 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/C, priority=13, startTime=1731991396356; duration=0sec 2024-11-19T04:43:16,814 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:16,814 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:C 2024-11-19T04:43:16,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:16,841 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111943276dfe309e494aa5bd36553c513a13_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111943276dfe309e494aa5bd36553c513a13_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:16,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/abff0c4b97384a7fbe45b29081159a99, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:16,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/abff0c4b97384a7fbe45b29081159a99 is 175, key is test_row_0/A:col10/1731991394734/Put/seqid=0 2024-11-19T04:43:16,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742427_1603 (size=31105) 2024-11-19T04:43:16,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991456960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991457067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991457177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991457177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,180 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,180 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_1' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,190 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1aa7847de13b4971ac03779aefac3b34 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1aa7847de13b4971ac03779aefac3b34 2024-11-19T04:43:17,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991457187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991457187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,191 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,191 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,195 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/A of 5eb489b71864e4cdf6076af86acc0211 into 1aa7847de13b4971ac03779aefac3b34(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:17,195 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:17,195 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/A, priority=13, startTime=1731991396356; duration=0sec 2024-11-19T04:43:17,195 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:17,195 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:A 2024-11-19T04:43:17,246 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/abff0c4b97384a7fbe45b29081159a99 2024-11-19T04:43:17,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/9f16222b84e84ac9819e6cc6c35b0196 is 50, key is test_row_0/B:col10/1731991394734/Put/seqid=0 2024-11-19T04:43:17,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742428_1604 (size=12151) 2024-11-19T04:43:17,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991457277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991457586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:17,657 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/9f16222b84e84ac9819e6cc6c35b0196 2024-11-19T04:43:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/1644cbd57c2942779fcba311e037a5ac is 50, key is test_row_0/C:col10/1731991394734/Put/seqid=0 2024-11-19T04:43:17,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742429_1605 (size=12151) 2024-11-19T04:43:18,069 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/1644cbd57c2942779fcba311e037a5ac 2024-11-19T04:43:18,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/abff0c4b97384a7fbe45b29081159a99 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99 2024-11-19T04:43:18,076 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99, entries=150, sequenceid=155, filesize=30.4 K 2024-11-19T04:43:18,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/9f16222b84e84ac9819e6cc6c35b0196 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9f16222b84e84ac9819e6cc6c35b0196 2024-11-19T04:43:18,079 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9f16222b84e84ac9819e6cc6c35b0196, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T04:43:18,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/1644cbd57c2942779fcba311e037a5ac as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/1644cbd57c2942779fcba311e037a5ac 2024-11-19T04:43:18,083 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/1644cbd57c2942779fcba311e037a5ac, entries=150, sequenceid=155, filesize=11.9 K 2024-11-19T04:43:18,083 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5eb489b71864e4cdf6076af86acc0211 in 1657ms, sequenceid=155, compaction requested=false 2024-11-19T04:43:18,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-19T04:43:18,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-19T04:43:18,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-19T04:43:18,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.4840 sec 2024-11-19T04:43:18,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 5.4890 sec 2024-11-19T04:43:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:18,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-19T04:43:18,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:18,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:18,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:18,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:18,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:18,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:18,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d82b490be2c486ebe4bd7effcaab053_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:18,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742430_1606 (size=14794) 2024-11-19T04:43:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991458146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:18,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991458256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:18,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991458467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:18,511 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:18,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195d82b490be2c486ebe4bd7effcaab053_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d82b490be2c486ebe4bd7effcaab053_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:18,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/b321de7af1be4808a252275777079a9d, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:18,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/b321de7af1be4808a252275777079a9d is 175, key is test_row_0/A:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742431_1607 (size=39749) 2024-11-19T04:43:18,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/b321de7af1be4808a252275777079a9d 2024-11-19T04:43:18,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/41a3d7ad93d94b4aac29afdbfa5b7d93 is 50, key is test_row_0/B:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:18,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742432_1608 (size=12151) 2024-11-19T04:43:18,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/41a3d7ad93d94b4aac29afdbfa5b7d93 2024-11-19T04:43:18,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/11a35d90fcf34a31bbca86a1d1157c15 is 50, key is test_row_0/C:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:18,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742433_1609 (size=12151) 2024-11-19T04:43:18,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991458777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:18,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/11a35d90fcf34a31bbca86a1d1157c15 2024-11-19T04:43:18,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/b321de7af1be4808a252275777079a9d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d 2024-11-19T04:43:18,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d, entries=200, sequenceid=176, filesize=38.8 K 2024-11-19T04:43:18,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/41a3d7ad93d94b4aac29afdbfa5b7d93 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/41a3d7ad93d94b4aac29afdbfa5b7d93 2024-11-19T04:43:18,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/41a3d7ad93d94b4aac29afdbfa5b7d93, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T04:43:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/11a35d90fcf34a31bbca86a1d1157c15 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/11a35d90fcf34a31bbca86a1d1157c15 2024-11-19T04:43:18,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/11a35d90fcf34a31bbca86a1d1157c15, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T04:43:18,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 5eb489b71864e4cdf6076af86acc0211 in 867ms, sequenceid=176, compaction requested=true 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:18,966 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:18,966 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:18,967 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:18,967 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:18,967 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/A is initiating minor compaction (all files) 2024-11-19T04:43:18,967 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/B is initiating minor compaction (all files) 2024-11-19T04:43:18,967 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/B in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:18,967 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/A in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:18,967 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1aa7847de13b4971ac03779aefac3b34, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=99.9 K 2024-11-19T04:43:18,967 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3792971d426f43a48ae680408a82a411, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9f16222b84e84ac9819e6cc6c35b0196, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/41a3d7ad93d94b4aac29afdbfa5b7d93] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=35.9 K 2024-11-19T04:43:18,967 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:18,967 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1aa7847de13b4971ac03779aefac3b34, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d] 2024-11-19T04:43:18,968 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 3792971d426f43a48ae680408a82a411, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731991393003 2024-11-19T04:43:18,968 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa7847de13b4971ac03779aefac3b34, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731991393003 2024-11-19T04:43:18,968 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f16222b84e84ac9819e6cc6c35b0196, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731991394734 2024-11-19T04:43:18,968 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting abff0c4b97384a7fbe45b29081159a99, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731991394734 2024-11-19T04:43:18,968 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting b321de7af1be4808a252275777079a9d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731991396949 2024-11-19T04:43:18,968 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a3d7ad93d94b4aac29afdbfa5b7d93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731991396949 2024-11-19T04:43:18,973 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:18,974 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:18,975 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195f86cecc83a94e6e8bf86ece8300b0d6_5eb489b71864e4cdf6076af86acc0211 store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:18,975 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/47f15d561f31450cb4872ba5a8324ab4 is 50, key is test_row_0/B:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:18,976 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195f86cecc83a94e6e8bf86ece8300b0d6_5eb489b71864e4cdf6076af86acc0211, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:18,976 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195f86cecc83a94e6e8bf86ece8300b0d6_5eb489b71864e4cdf6076af86acc0211 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:18,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742434_1610 (size=12561) 2024-11-19T04:43:18,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742435_1611 (size=4469) 2024-11-19T04:43:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:19,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:43:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:19,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194a612831acd1488b9538fd3b86c5388a_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991398140/Put/seqid=0 2024-11-19T04:43:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742436_1612 (size=12304) 2024-11-19T04:43:19,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991459370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:19,383 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/47f15d561f31450cb4872ba5a8324ab4 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/47f15d561f31450cb4872ba5a8324ab4 2024-11-19T04:43:19,385 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#A#compaction#515 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:19,386 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/e14430fa45ef4679a8b69c1810a8f654 is 175, key is test_row_0/A:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:19,386 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/B of 5eb489b71864e4cdf6076af86acc0211 into 47f15d561f31450cb4872ba5a8324ab4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:19,387 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:19,387 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/B, priority=13, startTime=1731991398966; duration=0sec 2024-11-19T04:43:19,387 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:19,387 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:B 2024-11-19T04:43:19,387 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:19,388 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:19,388 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/C is initiating minor compaction (all files) 2024-11-19T04:43:19,388 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/C in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:19,388 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/7be41dd437124c05a5e7223031947140, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/1644cbd57c2942779fcba311e037a5ac, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/11a35d90fcf34a31bbca86a1d1157c15] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=35.9 K 2024-11-19T04:43:19,389 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be41dd437124c05a5e7223031947140, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731991393003 2024-11-19T04:43:19,389 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 1644cbd57c2942779fcba311e037a5ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731991394734 2024-11-19T04:43:19,389 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a35d90fcf34a31bbca86a1d1157c15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731991396949 2024-11-19T04:43:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742437_1613 (size=31515) 2024-11-19T04:43:19,395 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#C#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:19,395 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/840b2758ef7b4a299814f5fc90bf3115 is 50, key is test_row_0/C:col10/1731991398098/Put/seqid=0 2024-11-19T04:43:19,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742438_1614 (size=12561) 2024-11-19T04:43:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991459477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991459687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:19,703 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:19,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194a612831acd1488b9538fd3b86c5388a_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194a612831acd1488b9538fd3b86c5388a_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:19,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/90cc51750847418ea2e4485a01a9b3f0, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:19,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/90cc51750847418ea2e4485a01a9b3f0 is 175, key is test_row_0/A:col10/1731991398140/Put/seqid=0 2024-11-19T04:43:19,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742439_1615 (size=31105) 2024-11-19T04:43:19,794 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/e14430fa45ef4679a8b69c1810a8f654 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/e14430fa45ef4679a8b69c1810a8f654 2024-11-19T04:43:19,798 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/A of 5eb489b71864e4cdf6076af86acc0211 into e14430fa45ef4679a8b69c1810a8f654(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:19,798 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:19,798 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/A, priority=13, startTime=1731991398966; duration=0sec 2024-11-19T04:43:19,798 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:19,799 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:A 2024-11-19T04:43:19,802 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/840b2758ef7b4a299814f5fc90bf3115 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/840b2758ef7b4a299814f5fc90bf3115 2024-11-19T04:43:19,805 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/C of 5eb489b71864e4cdf6076af86acc0211 into 840b2758ef7b4a299814f5fc90bf3115(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:19,805 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:19,805 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/C, priority=13, startTime=1731991398966; duration=0sec 2024-11-19T04:43:19,805 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:19,805 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:C 2024-11-19T04:43:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991459997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:20,112 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/90cc51750847418ea2e4485a01a9b3f0 2024-11-19T04:43:20,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/da10dc65fdeb4ac29a69f8a770907e3d is 50, key is test_row_0/B:col10/1731991398140/Put/seqid=0 2024-11-19T04:43:20,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742440_1616 (size=12151) 2024-11-19T04:43:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991460507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:20,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/da10dc65fdeb4ac29a69f8a770907e3d 2024-11-19T04:43:20,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/95e3d11f4c924b799ba603fdaeae67c2 is 50, key is test_row_0/C:col10/1731991398140/Put/seqid=0 2024-11-19T04:43:20,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742441_1617 (size=12151) 2024-11-19T04:43:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-19T04:43:20,807 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:20,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:43:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-19T04:43:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T04:43:20,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:43:20,810 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:43:20,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:43:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T04:43:20,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/95e3d11f4c924b799ba603fdaeae67c2 2024-11-19T04:43:20,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/90cc51750847418ea2e4485a01a9b3f0 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0 2024-11-19T04:43:20,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0, entries=150, sequenceid=190, filesize=30.4 K 2024-11-19T04:43:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/da10dc65fdeb4ac29a69f8a770907e3d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/da10dc65fdeb4ac29a69f8a770907e3d 2024-11-19T04:43:20,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/da10dc65fdeb4ac29a69f8a770907e3d, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T04:43:20,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/95e3d11f4c924b799ba603fdaeae67c2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/95e3d11f4c924b799ba603fdaeae67c2 2024-11-19T04:43:20,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/95e3d11f4c924b799ba603fdaeae67c2, entries=150, sequenceid=190, filesize=11.9 K 2024-11-19T04:43:20,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 5eb489b71864e4cdf6076af86acc0211 in 1659ms, sequenceid=190, compaction requested=false 2024-11-19T04:43:20,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:20,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-19T04:43:20,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:20,963 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-19T04:43:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:20,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119956b8c3eab744e4d83f93df8578b816a_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991399369/Put/seqid=0 2024-11-19T04:43:20,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742442_1618 (size=12304) 2024-11-19T04:43:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T04:43:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:21,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54898 deadline: 1731991461217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,223 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_2' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991461220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991461222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991461223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991461327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991461336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991461337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:21,385 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119956b8c3eab744e4d83f93df8578b816a_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119956b8c3eab744e4d83f93df8578b816a_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:21,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/3890869ab4a94a5780cf4130fbb4444e, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:21,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/3890869ab4a94a5780cf4130fbb4444e is 175, key is test_row_0/A:col10/1731991399369/Put/seqid=0 2024-11-19T04:43:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742443_1619 (size=31105) 2024-11-19T04:43:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T04:43:21,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991461517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,521 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991461536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991461547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991461547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,790 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/3890869ab4a94a5780cf4130fbb4444e 2024-11-19T04:43:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/98a0804c32494800b876ff5839d95796 is 50, key is test_row_0/B:col10/1731991399369/Put/seqid=0 2024-11-19T04:43:21,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742444_1620 (size=12151) 2024-11-19T04:43:21,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991461847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991461857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991461857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T04:43:22,205 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/98a0804c32494800b876ff5839d95796 2024-11-19T04:43:22,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/61f6b050c90648d29d692ba20ca65220 is 50, key is test_row_0/C:col10/1731991399369/Put/seqid=0 2024-11-19T04:43:22,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742445_1621 (size=12151) 2024-11-19T04:43:22,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991462357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991462366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991462367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:22,616 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/61f6b050c90648d29d692ba20ca65220 2024-11-19T04:43:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/3890869ab4a94a5780cf4130fbb4444e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e 2024-11-19T04:43:22,623 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e, entries=150, sequenceid=211, filesize=30.4 K 2024-11-19T04:43:22,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/98a0804c32494800b876ff5839d95796 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/98a0804c32494800b876ff5839d95796 2024-11-19T04:43:22,627 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/98a0804c32494800b876ff5839d95796, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T04:43:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/61f6b050c90648d29d692ba20ca65220 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f6b050c90648d29d692ba20ca65220 2024-11-19T04:43:22,631 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f6b050c90648d29d692ba20ca65220, entries=150, sequenceid=211, filesize=11.9 K 2024-11-19T04:43:22,632 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 5eb489b71864e4cdf6076af86acc0211 in 1669ms, sequenceid=211, compaction requested=true 2024-11-19T04:43:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-19T04:43:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-19T04:43:22,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-19T04:43:22,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-19T04:43:22,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8280 sec 2024-11-19T04:43:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-19T04:43:22,957 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:22,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:43:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-19T04:43:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T04:43:22,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:43:22,960 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:43:22,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:43:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T04:43:23,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-19T04:43:23,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:23,113 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-19T04:43:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:23,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f872cf9ba5c14c029289335f57600e04_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991401221/Put/seqid=0 2024-11-19T04:43:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742446_1622 (size=12304) 2024-11-19T04:43:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:23,129 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f872cf9ba5c14c029289335f57600e04_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f872cf9ba5c14c029289335f57600e04_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1a5f5562592647ada8a39a53dc1cc50f, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1a5f5562592647ada8a39a53dc1cc50f is 175, key is test_row_0/A:col10/1731991401221/Put/seqid=0 2024-11-19T04:43:23,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742447_1623 (size=31105) 2024-11-19T04:43:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T04:43:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:23,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991463407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991463414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991463415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991463516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991463526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991463527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,537 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1a5f5562592647ada8a39a53dc1cc50f 2024-11-19T04:43:23,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731991463537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,541 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 08a7f35e60d4,32975,1731991233019 for 'test_row_0' in 5eb489b71864e4cdf6076af86acc0211 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-19T04:43:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/0586c131a2f843e1be25a8b2c43be3fd is 50, key is test_row_0/B:col10/1731991401221/Put/seqid=0 2024-11-19T04:43:23,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742448_1624 (size=12151) 2024-11-19T04:43:23,550 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/0586c131a2f843e1be25a8b2c43be3fd 2024-11-19T04:43:23,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/de48a16edf3841269e947d1284b68b4d is 50, key is test_row_0/C:col10/1731991401221/Put/seqid=0 2024-11-19T04:43:23,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742449_1625 (size=12151) 2024-11-19T04:43:23,561 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/de48a16edf3841269e947d1284b68b4d 2024-11-19T04:43:23,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1a5f5562592647ada8a39a53dc1cc50f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f 2024-11-19T04:43:23,567 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f, entries=150, sequenceid=225, filesize=30.4 K 2024-11-19T04:43:23,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/0586c131a2f843e1be25a8b2c43be3fd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/0586c131a2f843e1be25a8b2c43be3fd 2024-11-19T04:43:23,571 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/0586c131a2f843e1be25a8b2c43be3fd, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T04:43:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/de48a16edf3841269e947d1284b68b4d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/de48a16edf3841269e947d1284b68b4d 2024-11-19T04:43:23,574 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/de48a16edf3841269e947d1284b68b4d, entries=150, sequenceid=225, filesize=11.9 K 2024-11-19T04:43:23,575 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 5eb489b71864e4cdf6076af86acc0211 in 462ms, sequenceid=225, compaction requested=true 2024-11-19T04:43:23,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:23,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:23,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-19T04:43:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-19T04:43:23,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-19T04:43:23,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-19T04:43:23,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 620 msec 2024-11-19T04:43:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-19T04:43:23,587 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:23,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T04:43:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-19T04:43:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:23,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T04:43:23,589 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T04:43:23,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T04:43:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:23,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-19T04:43:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:23,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:23,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fa68ea76a88c4e9699c4ceecc9be559e_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:23,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:23,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742450_1626 (size=12304) 2024-11-19T04:43:23,745 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:23,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fa68ea76a88c4e9699c4ceecc9be559e_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fa68ea76a88c4e9699c4ceecc9be559e_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:23,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/f5316bf0dac84f58926377c4a8b04148, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:23,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/f5316bf0dac84f58926377c4a8b04148 is 175, key is test_row_0/A:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742451_1627 (size=31105) 2024-11-19T04:43:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991463751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991463756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991463757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991463867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991463867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991463867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:23,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:23,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:23,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:24,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991464077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991464077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991464077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/f5316bf0dac84f58926377c4a8b04148 2024-11-19T04:43:24,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/9101c50c03244eafbd02d7b1f88a6469 is 50, key is test_row_0/B:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:24,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742452_1628 (size=12151) 2024-11-19T04:43:24,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:24,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991464387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991464387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991464387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/9101c50c03244eafbd02d7b1f88a6469 2024-11-19T04:43:24,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/9f3b3da1c2fb4380b910e98a372599dc is 50, key is test_row_0/C:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:24,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742453_1629 (size=12151) 2024-11-19T04:43:24,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:24,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991464897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991464897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991464897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:24,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:24,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:24,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T04:43:24,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/9f3b3da1c2fb4380b910e98a372599dc 2024-11-19T04:43:24,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/f5316bf0dac84f58926377c4a8b04148 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148 2024-11-19T04:43:24,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148, entries=150, sequenceid=245, filesize=30.4 K 2024-11-19T04:43:24,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/9101c50c03244eafbd02d7b1f88a6469 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9101c50c03244eafbd02d7b1f88a6469 2024-11-19T04:43:24,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9101c50c03244eafbd02d7b1f88a6469, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T04:43:24,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/9f3b3da1c2fb4380b910e98a372599dc as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f3b3da1c2fb4380b910e98a372599dc 2024-11-19T04:43:24,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f3b3da1c2fb4380b910e98a372599dc, entries=150, sequenceid=245, filesize=11.9 K 2024-11-19T04:43:24,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 5eb489b71864e4cdf6076af86acc0211 in 1260ms, sequenceid=245, compaction requested=true 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:24,990 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:43:24,990 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:24,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:24,991 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:43:24,991 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:43:24,991 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/A is initiating minor compaction (all files) 2024-11-19T04:43:24,991 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/B is initiating minor compaction (all files) 2024-11-19T04:43:24,991 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/A in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,991 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/B in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,991 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/47f15d561f31450cb4872ba5a8324ab4, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/da10dc65fdeb4ac29a69f8a770907e3d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/98a0804c32494800b876ff5839d95796, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/0586c131a2f843e1be25a8b2c43be3fd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9101c50c03244eafbd02d7b1f88a6469] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=59.7 K 2024-11-19T04:43:24,991 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/e14430fa45ef4679a8b69c1810a8f654, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=152.3 K 2024-11-19T04:43:24,991 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/e14430fa45ef4679a8b69c1810a8f654, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148] 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f15d561f31450cb4872ba5a8324ab4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731991396949 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting da10dc65fdeb4ac29a69f8a770907e3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991398140 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting e14430fa45ef4679a8b69c1810a8f654, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731991396949 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a0804c32494800b876ff5839d95796, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731991399318 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90cc51750847418ea2e4485a01a9b3f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991398140 2024-11-19T04:43:24,992 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 0586c131a2f843e1be25a8b2c43be3fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731991401209 2024-11-19T04:43:24,993 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3890869ab4a94a5780cf4130fbb4444e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731991399318 2024-11-19T04:43:24,993 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 9101c50c03244eafbd02d7b1f88a6469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731991403399 2024-11-19T04:43:24,993 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5f5562592647ada8a39a53dc1cc50f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731991401209 2024-11-19T04:43:24,993 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5316bf0dac84f58926377c4a8b04148, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731991403399 2024-11-19T04:43:25,001 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#B#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:25,001 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:25,001 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/3a7e119038b34380a60cb8f148877a71 is 50, key is test_row_0/B:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:25,002 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411192a903e76461e495e8f209ad60850471f_5eb489b71864e4cdf6076af86acc0211 store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:25,005 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411192a903e76461e495e8f209ad60850471f_5eb489b71864e4cdf6076af86acc0211, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:25,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742454_1630 (size=12731) 2024-11-19T04:43:25,005 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192a903e76461e495e8f209ad60850471f_5eb489b71864e4cdf6076af86acc0211 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:25,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742455_1631 (size=4469) 2024-11-19T04:43:25,009 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#A#compaction#531 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:25,009 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/ea0be6b1a098438e8a7afac251a85ddf is 175, key is test_row_0/A:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:25,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742456_1632 (size=31685) 2024-11-19T04:43:25,023 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/ea0be6b1a098438e8a7afac251a85ddf as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/ea0be6b1a098438e8a7afac251a85ddf 2024-11-19T04:43:25,027 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/A of 5eb489b71864e4cdf6076af86acc0211 into ea0be6b1a098438e8a7afac251a85ddf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:25,027 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:25,027 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/A, priority=11, startTime=1731991404990; duration=0sec 2024-11-19T04:43:25,027 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:25,027 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:A 2024-11-19T04:43:25,027 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-19T04:43:25,029 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-19T04:43:25,029 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/C is initiating minor compaction (all files) 2024-11-19T04:43:25,029 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/C in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:25,029 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/840b2758ef7b4a299814f5fc90bf3115, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/95e3d11f4c924b799ba603fdaeae67c2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f6b050c90648d29d692ba20ca65220, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/de48a16edf3841269e947d1284b68b4d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f3b3da1c2fb4380b910e98a372599dc] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=59.7 K 2024-11-19T04:43:25,029 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 840b2758ef7b4a299814f5fc90bf3115, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731991396949 2024-11-19T04:43:25,029 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95e3d11f4c924b799ba603fdaeae67c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731991398140 2024-11-19T04:43:25,030 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f6b050c90648d29d692ba20ca65220, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731991399318 2024-11-19T04:43:25,030 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting de48a16edf3841269e947d1284b68b4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731991401209 2024-11-19T04:43:25,030 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3b3da1c2fb4380b910e98a372599dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731991403399 2024-11-19T04:43:25,037 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#C#compaction#532 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:25,038 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/354585337acd411da758f171633dc9b2 is 50, key is test_row_0/C:col10/1731991403399/Put/seqid=0 2024-11-19T04:43:25,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742457_1633 (size=12731) 2024-11-19T04:43:25,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32975 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-19T04:43:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:25,117 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-19T04:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:25,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-19T04:43:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a5b5cdf1f4174afd8d5150c9edc65743_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991403754/Put/seqid=0 2024-11-19T04:43:25,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742458_1634 (size=12304) 2024-11-19T04:43:25,410 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/3a7e119038b34380a60cb8f148877a71 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3a7e119038b34380a60cb8f148877a71 2024-11-19T04:43:25,414 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/B of 5eb489b71864e4cdf6076af86acc0211 into 3a7e119038b34380a60cb8f148877a71(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:25,414 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:25,414 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/B, priority=11, startTime=1731991404990; duration=0sec 2024-11-19T04:43:25,414 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:25,414 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:B 2024-11-19T04:43:25,445 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/354585337acd411da758f171633dc9b2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/354585337acd411da758f171633dc9b2 2024-11-19T04:43:25,449 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/C of 5eb489b71864e4cdf6076af86acc0211 into 354585337acd411da758f171633dc9b2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:25,449 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:25,449 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/C, priority=11, startTime=1731991404990; duration=0sec 2024-11-19T04:43:25,449 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:25,449 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:C 2024-11-19T04:43:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:25,531 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a5b5cdf1f4174afd8d5150c9edc65743_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a5b5cdf1f4174afd8d5150c9edc65743_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:25,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ab34b71ae914abb9d97d13e535d2f2d, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:25,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ab34b71ae914abb9d97d13e535d2f2d is 175, key is test_row_0/A:col10/1731991403754/Put/seqid=0 2024-11-19T04:43:25,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742459_1635 (size=31105) 2024-11-19T04:43:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:25,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. as already flushing 2024-11-19T04:43:25,936 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ab34b71ae914abb9d97d13e535d2f2d 2024-11-19T04:43:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991465936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/8fb9b640d3304172be3d3fba3bfdd61f is 50, key is test_row_0/B:col10/1731991403754/Put/seqid=0 2024-11-19T04:43:25,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991465939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991465941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:25,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742460_1636 (size=12151) 2024-11-19T04:43:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991466047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991466047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991466057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991466257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991466257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991466267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,347 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/8fb9b640d3304172be3d3fba3bfdd61f 2024-11-19T04:43:26,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae is 50, key is test_row_0/C:col10/1731991403754/Put/seqid=0 2024-11-19T04:43:26,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742461_1637 (size=12151) 2024-11-19T04:43:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991466567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54834 deadline: 1731991466568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54862 deadline: 1731991466577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:26,762 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae 2024-11-19T04:43:26,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/8ab34b71ae914abb9d97d13e535d2f2d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d 2024-11-19T04:43:26,769 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d, entries=150, sequenceid=258, filesize=30.4 K 2024-11-19T04:43:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/8fb9b640d3304172be3d3fba3bfdd61f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8fb9b640d3304172be3d3fba3bfdd61f 2024-11-19T04:43:26,772 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8fb9b640d3304172be3d3fba3bfdd61f, entries=150, sequenceid=258, filesize=11.9 K 2024-11-19T04:43:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae 2024-11-19T04:43:26,776 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae, entries=150, sequenceid=258, filesize=11.9 K 2024-11-19T04:43:26,776 INFO [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5eb489b71864e4cdf6076af86acc0211 in 1659ms, sequenceid=258, compaction requested=false 2024-11-19T04:43:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/08a7f35e60d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-19T04:43:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-19T04:43:26,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-19T04:43:26,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1890 sec 2024-11-19T04:43:26,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 3.1930 sec 2024-11-19T04:43:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:27,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-19T04:43:27,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991405923, max timestamp=1731991407078Num uniques 859; 2024-11-19T04:43:27,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:27,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991405923, max timestamp=1731991407078Num uniques 859; 2024-11-19T04:43:27,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:27,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991405923, max timestamp=1731991407078Num uniques 859; 2024-11-19T04:43:27,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:27,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d90b2e67232e4c39967ca44d26cc45cb_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:27,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742462_1638 (size=14994) 2024-11-19T04:43:27,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731991407114, max timestamp=1731991407128Num uniques 859; 2024-11-19T04:43:27,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-19T04:43:27,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731991407114, max timestamp=1731991407128Num uniques 859; 2024-11-19T04:43:27,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731991407114, max timestamp=1731991407128Num uniques 859; 2024-11-19T04:43:27,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T04:43:27,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-19T04:43:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731991407167, max timestamp=1731991407190Num uniques 1018; 2024-11-19T04:43:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-19T04:43:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731991407167, max timestamp=1731991407190Num uniques 1009; 2024-11-19T04:43:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-19T04:43:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731991407167, max timestamp=1731991407190Num uniques 1009; 2024-11-19T04:43:27,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-19T04:43:27,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731991407224, max timestamp=1731991407261Num uniques 1077; 2024-11-19T04:43:27,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T04:43:27,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731991407224, max timestamp=1731991407261Num uniques 1077; 2024-11-19T04:43:27,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T04:43:27,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731991407224, max timestamp=1731991407261Num uniques 1077; 2024-11-19T04:43:27,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-19T04:43:27,307 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-11-19T04:43:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-19T04:43:27,307 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,308 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32975 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54824 deadline: 1731991467301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:27,308 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731991407283, max timestamp=1731991407305Num uniques 1086; 2024-11-19T04:43:27,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731991407283, max timestamp=1731991407305Num uniques 1086; 2024-11-19T04:43:27,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T04:43:27,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T04:43:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 , the old value is region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T04:43:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., hostname=08a7f35e60d4,32975,1731991233019, seqNum=5 because the exception is null or not the one we care about 2024-11-19T04:43:27,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731991407283, max timestamp=1731991407305Num uniques 1086; 2024-11-19T04:43:27,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/08a7f35e60d4:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-19T04:43:27,312 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-11-19T04:43:27,312 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,312 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,312 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,312 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-19T04:43:27,312 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,312 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,313 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,314 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-11-19T04:43:27,314 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,314 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,314 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,315 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-11-19T04:43:27,315 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,315 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,315 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,315 INFO [Thread-2503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2503. 2024-11-19T04:43:27,315 DEBUG [Thread-2503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,315 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,315 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,316 INFO [Thread-2501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2501. 2024-11-19T04:43:27,316 DEBUG [Thread-2501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,316 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,316 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,419 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-11-19T04:43:27,419 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,419 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,419 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,516 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:27,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d90b2e67232e4c39967ca44d26cc45cb_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d90b2e67232e4c39967ca44d26cc45cb_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:27,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/9503f17f6f614bf69680c32a74e56fcb, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:27,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/9503f17f6f614bf69680c32a74e56fcb is 175, key is test_row_0/A:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742463_1639 (size=39949) 2024-11-19T04:43:27,588 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-11-19T04:43:27,589 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:27,589 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,589 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:27,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-19T04:43:27,758 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:27,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/9503f17f6f614bf69680c32a74e56fcb 2024-11-19T04:43:27,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/4d46636dd4094fb3af403fb616cb0833 is 50, key is test_row_0/B:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742464_1640 (size=12301) 2024-11-19T04:43:28,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/4d46636dd4094fb3af403fb616cb0833 2024-11-19T04:43:28,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/51c4697b3ae9435a9fb3778a93252a0c is 50, key is test_row_0/C:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:28,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742465_1641 (size=12301) 2024-11-19T04:43:28,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/51c4697b3ae9435a9fb3778a93252a0c 2024-11-19T04:43:28,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/9503f17f6f614bf69680c32a74e56fcb as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb 2024-11-19T04:43:28,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb, entries=200, sequenceid=282, filesize=39.0 K 2024-11-19T04:43:28,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/4d46636dd4094fb3af403fb616cb0833 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/4d46636dd4094fb3af403fb616cb0833 2024-11-19T04:43:28,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/4d46636dd4094fb3af403fb616cb0833, entries=150, sequenceid=282, filesize=12.0 K 2024-11-19T04:43:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/51c4697b3ae9435a9fb3778a93252a0c as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/51c4697b3ae9435a9fb3778a93252a0c 2024-11-19T04:43:28,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/51c4697b3ae9435a9fb3778a93252a0c, entries=150, sequenceid=282, filesize=12.0 K 2024-11-19T04:43:28,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=53.67 KB/54960 for 5eb489b71864e4cdf6076af86acc0211 in 1678ms, sequenceid=282, compaction requested=true 2024-11-19T04:43:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T04:43:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T04:43:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:28,759 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:28,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eb489b71864e4cdf6076af86acc0211:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T04:43:28,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:28,760 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/B is initiating minor compaction (all files) 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/A is initiating minor compaction (all files) 2024-11-19T04:43:28,761 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/B in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:28,761 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/A in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:28,761 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3a7e119038b34380a60cb8f148877a71, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8fb9b640d3304172be3d3fba3bfdd61f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/4d46636dd4094fb3af403fb616cb0833] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=36.3 K 2024-11-19T04:43:28,761 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/ea0be6b1a098438e8a7afac251a85ddf, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=100.3 K 2024-11-19T04:43:28,761 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. files: [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/ea0be6b1a098438e8a7afac251a85ddf, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb] 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a7e119038b34380a60cb8f148877a71, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731991403399 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0be6b1a098438e8a7afac251a85ddf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731991403399 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb9b640d3304172be3d3fba3bfdd61f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731991403750 2024-11-19T04:43:28,761 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab34b71ae914abb9d97d13e535d2f2d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731991403750 2024-11-19T04:43:28,762 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9503f17f6f614bf69680c32a74e56fcb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731991405923 2024-11-19T04:43:28,762 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d46636dd4094fb3af403fb616cb0833, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731991405923 2024-11-19T04:43:28,767 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:28,767 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#B#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:28,768 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/502508dc25774755a74fb6f2e6bfa924 is 50, key is test_row_0/B:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:28,768 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fb52736fe2154a2bba0053fadf6511f5_5eb489b71864e4cdf6076af86acc0211 store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:28,773 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fb52736fe2154a2bba0053fadf6511f5_5eb489b71864e4cdf6076af86acc0211, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:28,774 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fb52736fe2154a2bba0053fadf6511f5_5eb489b71864e4cdf6076af86acc0211 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:28,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742466_1642 (size=12983) 2024-11-19T04:43:28,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742467_1643 (size=4469) 2024-11-19T04:43:28,783 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#A#compaction#540 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:28,783 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/81811445a3334ea8b655701a51a8405a is 175, key is test_row_0/A:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742468_1644 (size=31937) 2024-11-19T04:43:29,180 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/502508dc25774755a74fb6f2e6bfa924 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/502508dc25774755a74fb6f2e6bfa924 2024-11-19T04:43:29,183 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/B of 5eb489b71864e4cdf6076af86acc0211 into 502508dc25774755a74fb6f2e6bfa924(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:29,183 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:29,183 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/B, priority=13, startTime=1731991408759; duration=0sec 2024-11-19T04:43:29,183 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T04:43:29,183 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:B 2024-11-19T04:43:29,183 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T04:43:29,184 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T04:43:29,184 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1541): 5eb489b71864e4cdf6076af86acc0211/C is initiating minor compaction (all files) 2024-11-19T04:43:29,184 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eb489b71864e4cdf6076af86acc0211/C in TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:29,184 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/354585337acd411da758f171633dc9b2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/51c4697b3ae9435a9fb3778a93252a0c] into tmpdir=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp, totalSize=36.3 K 2024-11-19T04:43:29,184 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 354585337acd411da758f171633dc9b2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731991403399 2024-11-19T04:43:29,184 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac5a3e3e72b4e65936a03f5b4b0e3ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731991403750 2024-11-19T04:43:29,184 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c4697b3ae9435a9fb3778a93252a0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731991405923 2024-11-19T04:43:29,189 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/81811445a3334ea8b655701a51a8405a as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/81811445a3334ea8b655701a51a8405a 2024-11-19T04:43:29,190 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eb489b71864e4cdf6076af86acc0211#C#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T04:43:29,190 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/9f38652799bb45e098d8ecd7c1fef1b3 is 50, key is test_row_0/C:col10/1731991407079/Put/seqid=0 2024-11-19T04:43:29,192 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/A of 5eb489b71864e4cdf6076af86acc0211 into 81811445a3334ea8b655701a51a8405a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:29,193 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:29,193 INFO [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/A, priority=13, startTime=1731991408759; duration=0sec 2024-11-19T04:43:29,193 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:29,193 DEBUG [RS:0;08a7f35e60d4:32975-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:A 2024-11-19T04:43:29,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742469_1645 (size=12983) 2024-11-19T04:43:29,597 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/9f38652799bb45e098d8ecd7c1fef1b3 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f38652799bb45e098d8ecd7c1fef1b3 2024-11-19T04:43:29,600 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eb489b71864e4cdf6076af86acc0211/C of 5eb489b71864e4cdf6076af86acc0211 into 9f38652799bb45e098d8ecd7c1fef1b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T04:43:29,600 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:29,600 INFO [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211., storeName=5eb489b71864e4cdf6076af86acc0211/C, priority=13, startTime=1731991408759; duration=0sec 2024-11-19T04:43:29,600 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T04:43:29,600 DEBUG [RS:0;08a7f35e60d4:32975-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eb489b71864e4cdf6076af86acc0211:C 2024-11-19T04:43:31,219 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T04:43:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32975 {}] regionserver.HRegion(8855): Flush requested on 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:31,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eb489b71864e4cdf6076af86acc0211 3/3 column families, dataSize=60.38 KB heapSize=163.31 KB 2024-11-19T04:43:31,319 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-19T04:43:31,319 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T04:43:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=A 2024-11-19T04:43:31,319 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:31,319 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=B 2024-11-19T04:43:31,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5eb489b71864e4cdf6076af86acc0211, store=C 2024-11-19T04:43:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3107 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3118 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3220 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3103 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3100 2024-11-19T04:43:31,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T04:43:31,319 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:43:31,319 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:43:31,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:31,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:31,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:31,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T04:43:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:31,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T04:43:31,323 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991411323"}]},"ts":"1731991411323"} 2024-11-19T04:43:31,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dbb615ade816434999bed84abe57da0c_5eb489b71864e4cdf6076af86acc0211 is 50, key is test_row_0/A:col10/1731991407587/Put/seqid=0 2024-11-19T04:43:31,324 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T04:43:31,324 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T04:43:31,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T04:43:31,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, UNASSIGN}] 2024-11-19T04:43:31,327 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, UNASSIGN 2024-11-19T04:43:31,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=CLOSING, regionLocation=08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:31,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742470_1646 (size=12454) 2024-11-19T04:43:31,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, UNASSIGN because future has completed 2024-11-19T04:43:31,329 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T04:43:31,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019}] 2024-11-19T04:43:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T04:43:31,481 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:31,481 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T04:43:31,481 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 5eb489b71864e4cdf6076af86acc0211, disabling compactions & flushes 2024-11-19T04:43:31,481 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T04:43:31,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T04:43:31,731 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dbb615ade816434999bed84abe57da0c_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dbb615ade816434999bed84abe57da0c_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:31,731 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1dafc26e35b2468f8b4476c08d1ea435, store: [table=TestAcidGuarantees family=A region=5eb489b71864e4cdf6076af86acc0211] 2024-11-19T04:43:31,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1dafc26e35b2468f8b4476c08d1ea435 is 175, key is test_row_0/A:col10/1731991407587/Put/seqid=0 2024-11-19T04:43:31,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742471_1647 (size=31255) 2024-11-19T04:43:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T04:43:32,135 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1dafc26e35b2468f8b4476c08d1ea435 2024-11-19T04:43:32,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/998c98136642442c9bcbc7364b392efd is 50, key is test_row_0/B:col10/1731991407587/Put/seqid=0 2024-11-19T04:43:32,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742472_1648 (size=12301) 2024-11-19T04:43:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T04:43:32,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/998c98136642442c9bcbc7364b392efd 2024-11-19T04:43:32,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/09c8c9bc98d941e3a9f53216de49fdd2 is 50, key is test_row_0/C:col10/1731991407587/Put/seqid=0 2024-11-19T04:43:32,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742473_1649 (size=12301) 2024-11-19T04:43:32,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/09c8c9bc98d941e3a9f53216de49fdd2 2024-11-19T04:43:32,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/A/1dafc26e35b2468f8b4476c08d1ea435 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1dafc26e35b2468f8b4476c08d1ea435 2024-11-19T04:43:32,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1dafc26e35b2468f8b4476c08d1ea435, entries=150, sequenceid=362, filesize=30.5 K 2024-11-19T04:43:32,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/B/998c98136642442c9bcbc7364b392efd as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/998c98136642442c9bcbc7364b392efd 2024-11-19T04:43:32,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/998c98136642442c9bcbc7364b392efd, entries=150, sequenceid=362, filesize=12.0 K 2024-11-19T04:43:32,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/.tmp/C/09c8c9bc98d941e3a9f53216de49fdd2 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/09c8c9bc98d941e3a9f53216de49fdd2 2024-11-19T04:43:32,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/09c8c9bc98d941e3a9f53216de49fdd2, entries=150, sequenceid=362, filesize=12.0 K 2024-11-19T04:43:32,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~163.27 KB/167184, currentSize=0 B/0 for 5eb489b71864e4cdf6076af86acc0211 in 1648ms, sequenceid=362, compaction requested=false 2024-11-19T04:43:32,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eb489b71864e4cdf6076af86acc0211: 2024-11-19T04:43:32,967 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:32,967 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:32,967 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. after waiting 0 ms 2024-11-19T04:43:32,967 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:32,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/13c2fa61236b4d2bb5b6cd3b471fb4a3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/4d284ad67f1e40e8b89ddb46d3700145, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1aa7847de13b4971ac03779aefac3b34, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/e14430fa45ef4679a8b69c1810a8f654, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/ea0be6b1a098438e8a7afac251a85ddf, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb] to archive 2024-11-19T04:43:32,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:43:32,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/a681d0ca182642afa4b1aa8d31cdee85 2024-11-19T04:43:32,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/7888e7ac98294ac5a5ea928e6510e1d7 2024-11-19T04:43:32,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/c6570403225a456a8e2fc9f7e71b96c2 2024-11-19T04:43:32,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/13c2fa61236b4d2bb5b6cd3b471fb4a3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/13c2fa61236b4d2bb5b6cd3b471fb4a3 2024-11-19T04:43:32,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ff50d19878c452e86313c9a48fdd9c8 2024-11-19T04:43:32,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/375ef09b428a42409181631cdfbabce7 2024-11-19T04:43:32,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/4d284ad67f1e40e8b89ddb46d3700145 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/4d284ad67f1e40e8b89ddb46d3700145 2024-11-19T04:43:32,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/d985eaa02dd344278efabeb036a70bba 2024-11-19T04:43:32,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1d3a278e3d204f8a8e5f61487f0015c8 2024-11-19T04:43:32,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1aa7847de13b4971ac03779aefac3b34 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1aa7847de13b4971ac03779aefac3b34 2024-11-19T04:43:32,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/abff0c4b97384a7fbe45b29081159a99 2024-11-19T04:43:32,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/b321de7af1be4808a252275777079a9d 2024-11-19T04:43:32,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/e14430fa45ef4679a8b69c1810a8f654 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/e14430fa45ef4679a8b69c1810a8f654 2024-11-19T04:43:32,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/90cc51750847418ea2e4485a01a9b3f0 2024-11-19T04:43:32,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/3890869ab4a94a5780cf4130fbb4444e 2024-11-19T04:43:32,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1a5f5562592647ada8a39a53dc1cc50f 2024-11-19T04:43:32,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/ea0be6b1a098438e8a7afac251a85ddf to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/ea0be6b1a098438e8a7afac251a85ddf 2024-11-19T04:43:32,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/f5316bf0dac84f58926377c4a8b04148 2024-11-19T04:43:32,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/8ab34b71ae914abb9d97d13e535d2f2d 2024-11-19T04:43:32,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/9503f17f6f614bf69680c32a74e56fcb 2024-11-19T04:43:32,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/1046fa8bfd214bda95c06b90287db559, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/010776f760e94a3c8efc2f14bc4210c1, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6751e0e303d642df912d10546a392dbd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/335b17b47c6041eab862b7f83c8c69c8, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8938b5469d40465780a4f4c01ad120ec, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6ddaf75eb10f4638a0bbde9bd8df24ac, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/99a2913f8cae40e0836067de2e4eb867, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/b9e104fd60bd453ab53d71ea5a57d2d7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3792971d426f43a48ae680408a82a411, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/80e782629fad4081b1bc84b0f60c0841, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9f16222b84e84ac9819e6cc6c35b0196, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/47f15d561f31450cb4872ba5a8324ab4, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/41a3d7ad93d94b4aac29afdbfa5b7d93, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/da10dc65fdeb4ac29a69f8a770907e3d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/98a0804c32494800b876ff5839d95796, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/0586c131a2f843e1be25a8b2c43be3fd, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3a7e119038b34380a60cb8f148877a71, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9101c50c03244eafbd02d7b1f88a6469, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8fb9b640d3304172be3d3fba3bfdd61f, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/4d46636dd4094fb3af403fb616cb0833] to archive 2024-11-19T04:43:32,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:43:32,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/1046fa8bfd214bda95c06b90287db559 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/1046fa8bfd214bda95c06b90287db559 2024-11-19T04:43:32,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/010776f760e94a3c8efc2f14bc4210c1 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/010776f760e94a3c8efc2f14bc4210c1 2024-11-19T04:43:32,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6751e0e303d642df912d10546a392dbd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6751e0e303d642df912d10546a392dbd 2024-11-19T04:43:32,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/335b17b47c6041eab862b7f83c8c69c8 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/335b17b47c6041eab862b7f83c8c69c8 2024-11-19T04:43:32,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8938b5469d40465780a4f4c01ad120ec to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8938b5469d40465780a4f4c01ad120ec 2024-11-19T04:43:32,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6ddaf75eb10f4638a0bbde9bd8df24ac to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/6ddaf75eb10f4638a0bbde9bd8df24ac 2024-11-19T04:43:32,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/99a2913f8cae40e0836067de2e4eb867 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/99a2913f8cae40e0836067de2e4eb867 2024-11-19T04:43:32,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/b9e104fd60bd453ab53d71ea5a57d2d7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/b9e104fd60bd453ab53d71ea5a57d2d7 2024-11-19T04:43:32,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3792971d426f43a48ae680408a82a411 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3792971d426f43a48ae680408a82a411 2024-11-19T04:43:32,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/80e782629fad4081b1bc84b0f60c0841 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/80e782629fad4081b1bc84b0f60c0841 2024-11-19T04:43:32,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9f16222b84e84ac9819e6cc6c35b0196 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9f16222b84e84ac9819e6cc6c35b0196 2024-11-19T04:43:32,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/47f15d561f31450cb4872ba5a8324ab4 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/47f15d561f31450cb4872ba5a8324ab4 2024-11-19T04:43:32,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/41a3d7ad93d94b4aac29afdbfa5b7d93 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/41a3d7ad93d94b4aac29afdbfa5b7d93 2024-11-19T04:43:32,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/da10dc65fdeb4ac29a69f8a770907e3d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/da10dc65fdeb4ac29a69f8a770907e3d 2024-11-19T04:43:32,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/98a0804c32494800b876ff5839d95796 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/98a0804c32494800b876ff5839d95796 2024-11-19T04:43:32,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/0586c131a2f843e1be25a8b2c43be3fd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/0586c131a2f843e1be25a8b2c43be3fd 2024-11-19T04:43:32,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3a7e119038b34380a60cb8f148877a71 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/3a7e119038b34380a60cb8f148877a71 2024-11-19T04:43:32,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9101c50c03244eafbd02d7b1f88a6469 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/9101c50c03244eafbd02d7b1f88a6469 2024-11-19T04:43:32,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8fb9b640d3304172be3d3fba3bfdd61f to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/8fb9b640d3304172be3d3fba3bfdd61f 2024-11-19T04:43:32,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/4d46636dd4094fb3af403fb616cb0833 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/4d46636dd4094fb3af403fb616cb0833 2024-11-19T04:43:33,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/76fe7279604d45ff8ef9b0d771764e8b, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ace198dd688a4d1ca2e8b2f493fdc7ef, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/543641bb6f234715bbe5c70a5a1381b0, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f2ac82701d4a019de6c83672fdb6b3, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/afecce6685b2435fb8c0fae0dc1cf987, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/db6ca58b7e5f4375aaf16cfbc93b9a83, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ba3c2c9521cd49e797200b4f2f859de7, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/40387bbb0bfb44d1aa9aee12debe9f97, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/7be41dd437124c05a5e7223031947140, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/c26d05e8b7594305a08eb4f1cb2e235a, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/1644cbd57c2942779fcba311e037a5ac, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/840b2758ef7b4a299814f5fc90bf3115, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/11a35d90fcf34a31bbca86a1d1157c15, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/95e3d11f4c924b799ba603fdaeae67c2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f6b050c90648d29d692ba20ca65220, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/de48a16edf3841269e947d1284b68b4d, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/354585337acd411da758f171633dc9b2, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f3b3da1c2fb4380b910e98a372599dc, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/51c4697b3ae9435a9fb3778a93252a0c] to archive 2024-11-19T04:43:33,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T04:43:33,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/76fe7279604d45ff8ef9b0d771764e8b to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/76fe7279604d45ff8ef9b0d771764e8b 2024-11-19T04:43:33,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ace198dd688a4d1ca2e8b2f493fdc7ef to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ace198dd688a4d1ca2e8b2f493fdc7ef 2024-11-19T04:43:33,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/543641bb6f234715bbe5c70a5a1381b0 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/543641bb6f234715bbe5c70a5a1381b0 2024-11-19T04:43:33,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f2ac82701d4a019de6c83672fdb6b3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f2ac82701d4a019de6c83672fdb6b3 2024-11-19T04:43:33,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/afecce6685b2435fb8c0fae0dc1cf987 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/afecce6685b2435fb8c0fae0dc1cf987 2024-11-19T04:43:33,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/db6ca58b7e5f4375aaf16cfbc93b9a83 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/db6ca58b7e5f4375aaf16cfbc93b9a83 2024-11-19T04:43:33,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ba3c2c9521cd49e797200b4f2f859de7 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/ba3c2c9521cd49e797200b4f2f859de7 2024-11-19T04:43:33,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/40387bbb0bfb44d1aa9aee12debe9f97 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/40387bbb0bfb44d1aa9aee12debe9f97 2024-11-19T04:43:33,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/7be41dd437124c05a5e7223031947140 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/7be41dd437124c05a5e7223031947140 2024-11-19T04:43:33,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/c26d05e8b7594305a08eb4f1cb2e235a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/c26d05e8b7594305a08eb4f1cb2e235a 2024-11-19T04:43:33,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/1644cbd57c2942779fcba311e037a5ac to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/1644cbd57c2942779fcba311e037a5ac 2024-11-19T04:43:33,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/840b2758ef7b4a299814f5fc90bf3115 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/840b2758ef7b4a299814f5fc90bf3115 2024-11-19T04:43:33,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/11a35d90fcf34a31bbca86a1d1157c15 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/11a35d90fcf34a31bbca86a1d1157c15 2024-11-19T04:43:33,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/95e3d11f4c924b799ba603fdaeae67c2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/95e3d11f4c924b799ba603fdaeae67c2 2024-11-19T04:43:33,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f6b050c90648d29d692ba20ca65220 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/61f6b050c90648d29d692ba20ca65220 2024-11-19T04:43:33,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/de48a16edf3841269e947d1284b68b4d to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/de48a16edf3841269e947d1284b68b4d 2024-11-19T04:43:33,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/354585337acd411da758f171633dc9b2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/354585337acd411da758f171633dc9b2 2024-11-19T04:43:33,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f3b3da1c2fb4380b910e98a372599dc to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f3b3da1c2fb4380b910e98a372599dc 2024-11-19T04:43:33,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/5ac5a3e3e72b4e65936a03f5b4b0e3ae 2024-11-19T04:43:33,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/51c4697b3ae9435a9fb3778a93252a0c to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/51c4697b3ae9435a9fb3778a93252a0c 2024-11-19T04:43:33,020 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/recovered.edits/365.seqid, newMaxSeqId=365, maxSeqId=4 2024-11-19T04:43:33,020 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211. 2024-11-19T04:43:33,020 DEBUG [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 5eb489b71864e4cdf6076af86acc0211: Waiting for close lock at 1731991411481Running coprocessor pre-close hooks at 1731991411481Disabling compacts and flushes for region at 1731991411481Disabling writes for close at 1731991412967 (+1486 ms)Writing region close event to WAL at 1731991413017 (+50 ms)Running coprocessor post-close hooks at 1731991413020 (+3 ms)Closed at 1731991413020 2024-11-19T04:43:33,022 INFO [RS_CLOSE_REGION-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,022 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=5eb489b71864e4cdf6076af86acc0211, regionState=CLOSED 2024-11-19T04:43:33,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 because future has completed 2024-11-19T04:43:33,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-19T04:43:33,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 5eb489b71864e4cdf6076af86acc0211, server=08a7f35e60d4,32975,1731991233019 in 1.6950 sec 2024-11-19T04:43:33,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-19T04:43:33,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eb489b71864e4cdf6076af86acc0211, UNASSIGN in 1.6990 sec 2024-11-19T04:43:33,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-19T04:43:33,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7030 sec 2024-11-19T04:43:33,028 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731991413028"}]},"ts":"1731991413028"} 2024-11-19T04:43:33,029 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T04:43:33,029 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T04:43:33,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7100 sec 2024-11-19T04:43:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-19T04:43:33,467 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:33,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T04:43:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:33,469 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T04:43:33,470 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:33,471 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,473 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C, FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/recovered.edits] 2024-11-19T04:43:33,475 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1dafc26e35b2468f8b4476c08d1ea435 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/1dafc26e35b2468f8b4476c08d1ea435 2024-11-19T04:43:33,476 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/81811445a3334ea8b655701a51a8405a to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/A/81811445a3334ea8b655701a51a8405a 2024-11-19T04:43:33,477 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/502508dc25774755a74fb6f2e6bfa924 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/502508dc25774755a74fb6f2e6bfa924 2024-11-19T04:43:33,478 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/998c98136642442c9bcbc7364b392efd to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/B/998c98136642442c9bcbc7364b392efd 2024-11-19T04:43:33,480 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/09c8c9bc98d941e3a9f53216de49fdd2 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/09c8c9bc98d941e3a9f53216de49fdd2 2024-11-19T04:43:33,480 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f38652799bb45e098d8ecd7c1fef1b3 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/C/9f38652799bb45e098d8ecd7c1fef1b3 2024-11-19T04:43:33,482 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/recovered.edits/365.seqid to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211/recovered.edits/365.seqid 2024-11-19T04:43:33,483 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/default/TestAcidGuarantees/5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T04:43:33,483 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T04:43:33,484 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T04:43:33,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939b392ff3ca04d1e85607de574a7ce6f_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111939b392ff3ca04d1e85607de574a7ce6f_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111943276dfe309e494aa5bd36553c513a13_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111943276dfe309e494aa5bd36553c513a13_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,487 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119471095bec4544400987c70182555361c_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119471095bec4544400987c70182555361c_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,488 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194a612831acd1488b9538fd3b86c5388a_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194a612831acd1488b9538fd3b86c5388a_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,489 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111957937056ae18445e88ebb2e863ab5e29_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111957937056ae18445e88ebb2e863ab5e29_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,490 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d82b490be2c486ebe4bd7effcaab053_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195d82b490be2c486ebe4bd7effcaab053_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,490 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119956b8c3eab744e4d83f93df8578b816a_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119956b8c3eab744e4d83f93df8578b816a_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,491 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a5b5cdf1f4174afd8d5150c9edc65743_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a5b5cdf1f4174afd8d5150c9edc65743_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,492 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bd85ad16578546b4b69a03cef4b6efa1_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119bd85ad16578546b4b69a03cef4b6efa1_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,493 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d90b2e67232e4c39967ca44d26cc45cb_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d90b2e67232e4c39967ca44d26cc45cb_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,494 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dad785fb1c364eb39884bbcc9b623c7d_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dad785fb1c364eb39884bbcc9b623c7d_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,495 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dbb615ade816434999bed84abe57da0c_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dbb615ade816434999bed84abe57da0c_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,496 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119de88f1cfdf554243b40b651e8bdadb9b_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119de88f1cfdf554243b40b651e8bdadb9b_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,497 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dfd3f051e25b4722b71380e1e09b0540_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dfd3f051e25b4722b71380e1e09b0540_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f872cf9ba5c14c029289335f57600e04_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f872cf9ba5c14c029289335f57600e04_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fa68ea76a88c4e9699c4ceecc9be559e_5eb489b71864e4cdf6076af86acc0211 to hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fa68ea76a88c4e9699c4ceecc9be559e_5eb489b71864e4cdf6076af86acc0211 2024-11-19T04:43:33,499 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T04:43:33,501 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:33,503 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T04:43:33,505 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T04:43:33,506 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:33,506 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T04:43:33,506 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731991413506"}]},"ts":"9223372036854775807"} 2024-11-19T04:43:33,508 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T04:43:33,508 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5eb489b71864e4cdf6076af86acc0211, NAME => 'TestAcidGuarantees,,1731991384205.5eb489b71864e4cdf6076af86acc0211.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T04:43:33,508 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T04:43:33,508 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731991413508"}]},"ts":"9223372036854775807"} 2024-11-19T04:43:33,509 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T04:43:33,510 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T04:43:33,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-19T04:43:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-19T04:43:33,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T04:43:33,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T04:43:33,586 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 252), OpenFileDescriptor=447 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 362), ProcessCount=11 (was 11), AvailableMemoryMB=11418 (was 11492) 2024-11-19T04:43:33,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T04:43:33,586 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:43:33,586 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:43:33,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:33,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:33,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T04:43:33,587 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T04:43:33,587 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1166144013, stopped=false 2024-11-19T04:43:33,587 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=08a7f35e60d4,46599,1731991232199 2024-11-19T04:43:33,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T04:43:33,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T04:43:33,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:43:33,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:43:33,589 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T04:43:33,590 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T04:43:33,590 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T04:43:33,590 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:43:33,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:33,590 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T04:43:33,590 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '08a7f35e60d4,32975,1731991233019' ***** 2024-11-19T04:43:33,590 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T04:43:33,591 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T04:43:33,591 INFO [RS:0;08a7f35e60d4:32975 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T04:43:33,591 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T04:43:33,591 INFO [RS:0;08a7f35e60d4:32975 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T04:43:33,591 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(959): stopping server 08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:33,591 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T04:43:33,591 INFO [RS:0;08a7f35e60d4:32975 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;08a7f35e60d4:32975. 2024-11-19T04:43:33,591 DEBUG [RS:0;08a7f35e60d4:32975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T04:43:33,591 DEBUG [RS:0;08a7f35e60d4:32975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:33,592 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T04:43:33,592 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T04:43:33,592 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T04:43:33,592 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T04:43:33,592 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T04:43:33,592 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T04:43:33,593 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T04:43:33,593 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:33,593 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T04:43:33,593 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T04:43:33,593 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T04:43:33,593 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T04:43:33,593 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T04:43:33,601 INFO [regionserver/08a7f35e60d4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T04:43:33,617 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/info/5202d017c4f24959855c71d065ee3c3d is 95, key is TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad./info:/1731991263424/DeleteFamily/seqid=0 2024-11-19T04:43:33,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742474_1650 (size=6514) 2024-11-19T04:43:33,793 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:33,993 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:34,021 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/info/5202d017c4f24959855c71d065ee3c3d 2024-11-19T04:43:34,038 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/ns/f777859f3c6649a0b86c94d9210a4553 is 93, key is TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad./ns:/1731991263361/DeleteFamily/seqid=0 2024-11-19T04:43:34,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742475_1651 (size=5906) 2024-11-19T04:43:34,108 INFO [regionserver/08a7f35e60d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T04:43:34,108 INFO [regionserver/08a7f35e60d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T04:43:34,193 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:34,393 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:34,442 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/ns/f777859f3c6649a0b86c94d9210a4553 2024-11-19T04:43:34,458 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/rep_barrier/4755c5114cad42a9b88b3baf1076529e is 102, key is TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad./rep_barrier:/1731991263361/DeleteFamily/seqid=0 2024-11-19T04:43:34,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742476_1652 (size=6025) 2024-11-19T04:43:34,594 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:34,794 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:34,862 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/rep_barrier/4755c5114cad42a9b88b3baf1076529e 2024-11-19T04:43:34,879 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/table/6ebf3e056bbc4089b76906e89cde66c9 is 96, key is TestAcidGuarantees,,1731991236748.e97938390deb1164b875a27699276aad./table:/1731991263361/DeleteFamily/seqid=0 2024-11-19T04:43:34,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742477_1653 (size=5936) 2024-11-19T04:43:34,994 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:35,178 INFO [regionserver/08a7f35e60d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-19T04:43:35,194 DEBUG [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T04:43:35,283 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/table/6ebf3e056bbc4089b76906e89cde66c9 2024-11-19T04:43:35,286 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/info/5202d017c4f24959855c71d065ee3c3d as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/info/5202d017c4f24959855c71d065ee3c3d 2024-11-19T04:43:35,289 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/info/5202d017c4f24959855c71d065ee3c3d, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T04:43:35,290 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/ns/f777859f3c6649a0b86c94d9210a4553 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/ns/f777859f3c6649a0b86c94d9210a4553 2024-11-19T04:43:35,292 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/ns/f777859f3c6649a0b86c94d9210a4553, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T04:43:35,293 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/rep_barrier/4755c5114cad42a9b88b3baf1076529e as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/rep_barrier/4755c5114cad42a9b88b3baf1076529e 2024-11-19T04:43:35,295 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/rep_barrier/4755c5114cad42a9b88b3baf1076529e, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T04:43:35,295 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/.tmp/table/6ebf3e056bbc4089b76906e89cde66c9 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/table/6ebf3e056bbc4089b76906e89cde66c9 2024-11-19T04:43:35,298 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/table/6ebf3e056bbc4089b76906e89cde66c9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T04:43:35,298 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1705ms, sequenceid=90, compaction requested=false 2024-11-19T04:43:35,302 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T04:43:35,302 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T04:43:35,302 INFO [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T04:43:35,303 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731991413592Running coprocessor pre-close hooks at 1731991413593 (+1 ms)Disabling compacts and flushes for region at 1731991413593Disabling writes for close at 1731991413593Obtaining lock to block concurrent updates at 1731991413593Preparing flush snapshotting stores in 1588230740 at 1731991413593Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731991413593Flushing stores of hbase:meta,,1.1588230740 at 1731991413594 (+1 ms)Flushing 1588230740/info: creating writer at 1731991413594Flushing 1588230740/info: appending metadata at 1731991413616 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731991413616Flushing 1588230740/ns: creating writer at 1731991414024 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731991414038 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731991414038Flushing 1588230740/rep_barrier: creating writer at 1731991414444 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731991414458 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731991414458Flushing 1588230740/table: creating writer at 1731991414865 (+407 ms)Flushing 1588230740/table: appending metadata at 1731991414879 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731991414879Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28f54223: reopening flushed file at 1731991415286 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63722609: reopening flushed file at 1731991415289 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19c12862: reopening flushed file at 1731991415292 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a8727be: reopening flushed file at 1731991415295 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1705ms, sequenceid=90, compaction requested=false at 1731991415298 (+3 ms)Writing region close event to WAL at 1731991415300 (+2 ms)Running coprocessor post-close hooks at 1731991415302 (+2 ms)Closed at 1731991415302 2024-11-19T04:43:35,303 DEBUG [RS_CLOSE_META-regionserver/08a7f35e60d4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T04:43:35,395 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(976): stopping server 08a7f35e60d4,32975,1731991233019; all regions closed. 2024-11-19T04:43:35,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741834_1010 (size=25344) 2024-11-19T04:43:35,401 DEBUG [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/oldWALs 2024-11-19T04:43:35,401 INFO [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 08a7f35e60d4%2C32975%2C1731991233019.meta:.meta(num 1731991236266) 2024-11-19T04:43:35,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741832_1008 (size=18985357) 2024-11-19T04:43:35,405 DEBUG [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/oldWALs 2024-11-19T04:43:35,405 INFO [RS:0;08a7f35e60d4:32975 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 08a7f35e60d4%2C32975%2C1731991233019:(num 1731991235269) 2024-11-19T04:43:35,405 DEBUG [RS:0;08a7f35e60d4:32975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T04:43:35,405 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T04:43:35,405 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T04:43:35,405 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.ChoreService(370): Chore service for: regionserver/08a7f35e60d4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T04:43:35,405 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T04:43:35,405 INFO [regionserver/08a7f35e60d4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T04:43:35,406 INFO [RS:0;08a7f35e60d4:32975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32975 2024-11-19T04:43:35,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/08a7f35e60d4,32975,1731991233019 2024-11-19T04:43:35,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T04:43:35,410 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T04:43:35,411 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [08a7f35e60d4,32975,1731991233019] 2024-11-19T04:43:35,414 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/08a7f35e60d4,32975,1731991233019 already deleted, retry=false 2024-11-19T04:43:35,415 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 08a7f35e60d4,32975,1731991233019 expired; onlineServers=0 2024-11-19T04:43:35,415 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '08a7f35e60d4,46599,1731991232199' ***** 2024-11-19T04:43:35,415 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T04:43:35,415 INFO [M:0;08a7f35e60d4:46599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T04:43:35,415 INFO [M:0;08a7f35e60d4:46599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T04:43:35,415 DEBUG [M:0;08a7f35e60d4:46599 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T04:43:35,415 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T04:43:35,415 DEBUG [M:0;08a7f35e60d4:46599 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T04:43:35,415 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster-HFileCleaner.large.0-1731991235020 {}] cleaner.HFileCleaner(306): Exit Thread[master/08a7f35e60d4:0:becomeActiveMaster-HFileCleaner.large.0-1731991235020,5,FailOnTimeoutGroup] 2024-11-19T04:43:35,415 DEBUG [master/08a7f35e60d4:0:becomeActiveMaster-HFileCleaner.small.0-1731991235020 {}] cleaner.HFileCleaner(306): Exit Thread[master/08a7f35e60d4:0:becomeActiveMaster-HFileCleaner.small.0-1731991235020,5,FailOnTimeoutGroup] 2024-11-19T04:43:35,415 INFO [M:0;08a7f35e60d4:46599 {}] hbase.ChoreService(370): Chore service for: master/08a7f35e60d4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T04:43:35,416 INFO [M:0;08a7f35e60d4:46599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T04:43:35,416 DEBUG [M:0;08a7f35e60d4:46599 {}] master.HMaster(1795): Stopping service threads 2024-11-19T04:43:35,416 INFO [M:0;08a7f35e60d4:46599 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T04:43:35,416 INFO [M:0;08a7f35e60d4:46599 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T04:43:35,416 ERROR [M:0;08a7f35e60d4:46599 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T04:43:35,417 INFO [M:0;08a7f35e60d4:46599 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T04:43:35,417 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T04:43:35,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T04:43:35,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T04:43:35,418 DEBUG [M:0;08a7f35e60d4:46599 {}] zookeeper.ZKUtil(347): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T04:43:35,418 WARN [M:0;08a7f35e60d4:46599 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T04:43:35,418 INFO [M:0;08a7f35e60d4:46599 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/.lastflushedseqids 2024-11-19T04:43:35,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742478_1654 (size=72) 2024-11-19T04:43:35,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T04:43:35,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32975-0x1012e86c5d20001, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T04:43:35,513 INFO [RS:0;08a7f35e60d4:32975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T04:43:35,513 INFO [RS:0;08a7f35e60d4:32975 {}] regionserver.HRegionServer(1031): Exiting; stopping=08a7f35e60d4,32975,1731991233019; zookeeper connection closed. 2024-11-19T04:43:35,514 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1746226c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1746226c 2024-11-19T04:43:35,514 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T04:43:35,829 INFO [M:0;08a7f35e60d4:46599 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T04:43:35,829 INFO [M:0;08a7f35e60d4:46599 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T04:43:35,829 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T04:43:35,829 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:43:35,829 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:43:35,829 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T04:43:35,829 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:43:35,829 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=786.28 KB heapSize=963.39 KB 2024-11-19T04:43:35,845 DEBUG [M:0;08a7f35e60d4:46599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1c45546dea374e258545c2721a6f3c4f is 82, key is hbase:meta,,1/info:regioninfo/1731991236350/Put/seqid=0 2024-11-19T04:43:35,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742479_1655 (size=5672) 2024-11-19T04:43:36,248 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2185 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1c45546dea374e258545c2721a6f3c4f 2024-11-19T04:43:36,268 DEBUG [M:0;08a7f35e60d4:46599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5070b0e8e29436fa6cc356514d5ea19 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1731991387271/Put/seqid=0 2024-11-19T04:43:36,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742480_1656 (size=46408) 2024-11-19T04:43:36,672 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785.72 KB at sequenceid=2185 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5070b0e8e29436fa6cc356514d5ea19 2024-11-19T04:43:36,675 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b5070b0e8e29436fa6cc356514d5ea19 2024-11-19T04:43:36,690 DEBUG [M:0;08a7f35e60d4:46599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2443020287424b8a9ffd88dd3eecbe82 is 69, key is 08a7f35e60d4,32975,1731991233019/rs:state/1731991235030/Put/seqid=0 2024-11-19T04:43:36,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073742481_1657 (size=5156) 2024-11-19T04:43:37,093 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2185 (bloomFilter=true), to=hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2443020287424b8a9ffd88dd3eecbe82 2024-11-19T04:43:37,096 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1c45546dea374e258545c2721a6f3c4f as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1c45546dea374e258545c2721a6f3c4f 2024-11-19T04:43:37,099 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1c45546dea374e258545c2721a6f3c4f, entries=8, sequenceid=2185, filesize=5.5 K 2024-11-19T04:43:37,100 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5070b0e8e29436fa6cc356514d5ea19 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b5070b0e8e29436fa6cc356514d5ea19 2024-11-19T04:43:37,102 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b5070b0e8e29436fa6cc356514d5ea19 2024-11-19T04:43:37,102 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b5070b0e8e29436fa6cc356514d5ea19, entries=176, sequenceid=2185, filesize=45.3 K 2024-11-19T04:43:37,103 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2443020287424b8a9ffd88dd3eecbe82 as hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2443020287424b8a9ffd88dd3eecbe82 2024-11-19T04:43:37,105 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34817/user/jenkins/test-data/7b7a3a3d-4fce-fdbf-7658-1389076bdd04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2443020287424b8a9ffd88dd3eecbe82, entries=1, sequenceid=2185, filesize=5.0 K 2024-11-19T04:43:37,106 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(3140): Finished flush of dataSize ~786.28 KB/805149, heapSize ~963.09 KB/986208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2185, compaction requested=false 2024-11-19T04:43:37,107 INFO [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T04:43:37,107 DEBUG [M:0;08a7f35e60d4:46599 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731991415829Disabling compacts and flushes for region at 1731991415829Disabling writes for close at 1731991415829Obtaining lock to block concurrent updates at 1731991415829Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731991415829Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=805149, getHeapSize=986448, getOffHeapSize=0, getCellsCount=2386 at 1731991415830 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731991415830Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731991415830Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731991415845 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731991415845Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731991416252 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731991416267 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731991416267Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731991416675 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731991416689 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731991416689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18bc58d1: reopening flushed file at 1731991417096 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ecf2dbc: reopening flushed file at 1731991417099 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a578dc: reopening flushed file at 1731991417102 (+3 ms)Finished flush of dataSize ~786.28 KB/805149, heapSize ~963.09 KB/986208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2185, compaction requested=false at 1731991417106 (+4 ms)Writing region close event to WAL at 1731991417107 (+1 ms)Closed at 1731991417107 2024-11-19T04:43:37,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38523 is added to blk_1073741830_1006 (size=948397) 2024-11-19T04:43:37,109 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T04:43:37,110 INFO [M:0;08a7f35e60d4:46599 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T04:43:37,110 INFO [M:0;08a7f35e60d4:46599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46599 2024-11-19T04:43:37,110 INFO [M:0;08a7f35e60d4:46599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T04:43:37,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T04:43:37,212 INFO [M:0;08a7f35e60d4:46599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T04:43:37,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46599-0x1012e86c5d20000, quorum=127.0.0.1:56501, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T04:43:37,216 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T04:43:37,218 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T04:43:37,218 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T04:43:37,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T04:43:37,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/hadoop.log.dir/,STOPPED} 2024-11-19T04:43:37,221 WARN [BP-1974752605-172.17.0.2-1731991228892 heartbeating to localhost/127.0.0.1:34817 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T04:43:37,221 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T04:43:37,221 WARN [BP-1974752605-172.17.0.2-1731991228892 heartbeating to localhost/127.0.0.1:34817 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1974752605-172.17.0.2-1731991228892 (Datanode Uuid 5cca3b6b-f189-46f5-842b-0fb0bedb6747) service to localhost/127.0.0.1:34817 2024-11-19T04:43:37,222 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T04:43:37,223 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/data/data1/current/BP-1974752605-172.17.0.2-1731991228892 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T04:43:37,224 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/cluster_1b6b9e8f-d71a-ec44-fa37-488727bd99f3/data/data2/current/BP-1974752605-172.17.0.2-1731991228892 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T04:43:37,224 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T04:43:37,232 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T04:43:37,232 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T04:43:37,232 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T04:43:37,232 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T04:43:37,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/25206d33-35cb-2b1f-8fc5-d887a6555f4c/hadoop.log.dir/,STOPPED} 2024-11-19T04:43:37,249 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T04:43:37,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down